rainfallqc 0.2.4__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/PKG-INFO +20 -29
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/README.rst +19 -28
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/pyproject.toml +1 -1
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/__init__.py +1 -1
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/data_readers.py +80 -80
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/data_utils.py +2 -2
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/LICENSE +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/comparison_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/gauge_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/neighbourhood_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/pypwsqc_filters.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/checks/timeseries_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/core/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/core/all_qc_checks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_CDD_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_CWD_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_PRCPTOT_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_R99p_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_Rx1day_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/RawData_HADEX2_SDII_1951-2010_ANN_from-90to90_from-180to180.nc +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/ETCCDI/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/data/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/qc_frameworks/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/qc_frameworks/apply_qc_framework.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/qc_frameworks/inbuilt_qc_frameworks.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/__init__.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/neighbourhood_utils.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/spatial_utils.py +0 -0
- {rainfallqc-0.2.4 → rainfallqc-0.2.5}/src/rainfallqc/utils/stats.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rainfallqc
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Quality control for rainfall data
|
|
5
5
|
Author-email: Tom Keel <tomkee@ceh.ac.uk>
|
|
6
6
|
Maintainer-email: Tom Keel <tomkee@ceh.ac.uk>
|
|
@@ -92,49 +92,41 @@ Example 1. - Individual quality checks on single rain gauge
|
|
|
92
92
|
|
|
93
93
|
.. code-block:: python
|
|
94
94
|
|
|
95
|
-
# Load two types of QC'ing modules from RainfallQC
|
|
96
95
|
from rainfallqc import gauge_checks, comparison_checks
|
|
97
96
|
|
|
98
|
-
|
|
99
|
-
# 1.1. Run 1 qc method for 1 gauge
|
|
97
|
+
data = pl.read_csv("rain_gauge_data.csv")
|
|
100
98
|
intermittency_flag = gauge_checks.check_intermittency(data, target_gauge_col="rain_mm")
|
|
101
99
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
# 1.3. Run 1 qc method for 1 gauge using in-built comparison dataset and location of gauge
|
|
106
|
-
rx1day_flags = comparison_checks.check_annual_exceedance_etccdi_rx1day(data, target_gauge_col="rain_mm", gauge_lon=1.0, gauge_lat=55.0)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
Example 2. - Individual quality checks on networks of rain gauges
|
|
110
|
-
=================================================================
|
|
100
|
+
Example 2. - Neighbourhood quality checks for the global sub-daily rain gauge network (GSDR)
|
|
101
|
+
============================================================================================
|
|
111
102
|
|
|
112
103
|
.. code-block:: python
|
|
113
104
|
|
|
114
|
-
# 2. Run neighbour/network checks on a subset of a rain gauge network
|
|
115
105
|
from rainfallqc import neighbourhood_checks
|
|
116
106
|
from rainfallqc.utils import data_readers
|
|
117
107
|
|
|
118
|
-
|
|
119
|
-
|
|
108
|
+
distance_threshold = 50 # km
|
|
109
|
+
n_closest = 10 # number of closest neighbours to consider
|
|
110
|
+
min_overlap_days = 500 # minimum overlapping days to be considered a neighbour
|
|
111
|
+
|
|
112
|
+
gsdr_obj = data_readers.GSDRNetworkReader(path_to_gsdr_dir="path/to/GSDR/data")
|
|
120
113
|
|
|
121
|
-
# 2.2. subset by max 10 gauges within 50 km and with at least 500 days of overlap
|
|
122
114
|
nearby_ids = list(
|
|
123
|
-
|
|
124
|
-
target_id="DE_00310", distance_threshold=
|
|
115
|
+
gsdr_obj.get_nearest_overlapping_neighbours_to_target(
|
|
116
|
+
target_id="DE_00310", distance_threshold=distance_threshold, n_closest=n_closest, min_overlap_days=min_overlap_days
|
|
125
117
|
)
|
|
126
118
|
)
|
|
127
119
|
nearby_ids.append(target_id)
|
|
128
|
-
nearby_data_paths =
|
|
120
|
+
nearby_data_paths = gsdr_obj.metadata.filter(pl.col("station_id").is_in(nearby_ids))["path"]
|
|
129
121
|
|
|
130
|
-
#
|
|
131
|
-
|
|
122
|
+
# Load those nearest gauges from network metadata
|
|
123
|
+
gsdr_network = gsdr_obj.load_network_data(data_paths=nearby_data_paths)
|
|
132
124
|
|
|
133
|
-
#
|
|
125
|
+
# Run wet neighbour check
|
|
134
126
|
extreme_wet_flags = neighbourhood_checks.check_wet_neighbours(
|
|
135
|
-
|
|
127
|
+
gsdr_network,
|
|
136
128
|
target_gauge_col="rain_mm_DE_02483",
|
|
137
|
-
neighbouring_gauge_cols=
|
|
129
|
+
neighbouring_gauge_cols=gsdr_network.columns[1:], # exclude time
|
|
138
130
|
time_res="hourly",
|
|
139
131
|
wet_threshold=1.0, # threshold for rainfall intensity to be considered
|
|
140
132
|
min_n_neighbours=5, # number of neighbours needed for comparison
|
|
@@ -146,14 +138,13 @@ Example 3. - Applying a framework of QC methods (e.g. IntenseQC)
|
|
|
146
138
|
|
|
147
139
|
.. code-block:: python
|
|
148
140
|
|
|
149
|
-
# 3. Applying multiple QC methods in framework (e.g. IntenseQC)
|
|
150
141
|
from rainfallqc.qc_frameworks import apply_qc_framework
|
|
151
142
|
|
|
152
|
-
#
|
|
143
|
+
# 1. Decide which QC methods of IntenseQC will be run
|
|
153
144
|
qc_framework = "IntenseQC"
|
|
154
145
|
qc_methods_to_run = ["QC1", "QC8", "QC9", "QC10", "QC11", "QC12", "QC14", "QC15", "QC16"]
|
|
155
146
|
|
|
156
|
-
#
|
|
147
|
+
# 2 Decide which parameters for QC
|
|
157
148
|
qc_kwargs = {
|
|
158
149
|
"QC1": {"quantile": 5},
|
|
159
150
|
"QC14": {"wet_day_threshold": 1.0, "accumulation_multiplying_factor": 2.0},
|
|
@@ -173,7 +164,7 @@ Example 3. - Applying a framework of QC methods (e.g. IntenseQC)
|
|
|
173
164
|
},
|
|
174
165
|
}
|
|
175
166
|
|
|
176
|
-
# 3.
|
|
167
|
+
# 3. Run QC methods on network data
|
|
177
168
|
qc_result = apply_qc_framework.run_qc_framework(
|
|
178
169
|
daily_gpcc_network, qc_framework=qc_framework, qc_methods_to_run=qc_methods_to_run, qc_kwargs=qc_kwargs
|
|
179
170
|
)
|
|
@@ -34,49 +34,41 @@ Example 1. - Individual quality checks on single rain gauge
|
|
|
34
34
|
|
|
35
35
|
.. code-block:: python
|
|
36
36
|
|
|
37
|
-
# Load two types of QC'ing modules from RainfallQC
|
|
38
37
|
from rainfallqc import gauge_checks, comparison_checks
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
# 1.1. Run 1 qc method for 1 gauge
|
|
39
|
+
data = pl.read_csv("rain_gauge_data.csv")
|
|
42
40
|
intermittency_flag = gauge_checks.check_intermittency(data, target_gauge_col="rain_mm")
|
|
43
41
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
# 1.3. Run 1 qc method for 1 gauge using in-built comparison dataset and location of gauge
|
|
48
|
-
rx1day_flags = comparison_checks.check_annual_exceedance_etccdi_rx1day(data, target_gauge_col="rain_mm", gauge_lon=1.0, gauge_lat=55.0)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
Example 2. - Individual quality checks on networks of rain gauges
|
|
52
|
-
=================================================================
|
|
42
|
+
Example 2. - Neighbourhood quality checks for the global sub-daily rain gauge network (GSDR)
|
|
43
|
+
============================================================================================
|
|
53
44
|
|
|
54
45
|
.. code-block:: python
|
|
55
46
|
|
|
56
|
-
# 2. Run neighbour/network checks on a subset of a rain gauge network
|
|
57
47
|
from rainfallqc import neighbourhood_checks
|
|
58
48
|
from rainfallqc.utils import data_readers
|
|
59
49
|
|
|
60
|
-
|
|
61
|
-
|
|
50
|
+
distance_threshold = 50 # km
|
|
51
|
+
n_closest = 10 # number of closest neighbours to consider
|
|
52
|
+
min_overlap_days = 500 # minimum overlapping days to be considered a neighbour
|
|
53
|
+
|
|
54
|
+
gsdr_obj = data_readers.GSDRNetworkReader(path_to_gsdr_dir="path/to/GSDR/data")
|
|
62
55
|
|
|
63
|
-
# 2.2. subset by max 10 gauges within 50 km and with at least 500 days of overlap
|
|
64
56
|
nearby_ids = list(
|
|
65
|
-
|
|
66
|
-
target_id="DE_00310", distance_threshold=
|
|
57
|
+
gsdr_obj.get_nearest_overlapping_neighbours_to_target(
|
|
58
|
+
target_id="DE_00310", distance_threshold=distance_threshold, n_closest=n_closest, min_overlap_days=min_overlap_days
|
|
67
59
|
)
|
|
68
60
|
)
|
|
69
61
|
nearby_ids.append(target_id)
|
|
70
|
-
nearby_data_paths =
|
|
62
|
+
nearby_data_paths = gsdr_obj.metadata.filter(pl.col("station_id").is_in(nearby_ids))["path"]
|
|
71
63
|
|
|
72
|
-
#
|
|
73
|
-
|
|
64
|
+
# Load those nearest gauges from network metadata
|
|
65
|
+
gsdr_network = gsdr_obj.load_network_data(data_paths=nearby_data_paths)
|
|
74
66
|
|
|
75
|
-
#
|
|
67
|
+
# Run wet neighbour check
|
|
76
68
|
extreme_wet_flags = neighbourhood_checks.check_wet_neighbours(
|
|
77
|
-
|
|
69
|
+
gsdr_network,
|
|
78
70
|
target_gauge_col="rain_mm_DE_02483",
|
|
79
|
-
neighbouring_gauge_cols=
|
|
71
|
+
neighbouring_gauge_cols=gsdr_network.columns[1:], # exclude time
|
|
80
72
|
time_res="hourly",
|
|
81
73
|
wet_threshold=1.0, # threshold for rainfall intensity to be considered
|
|
82
74
|
min_n_neighbours=5, # number of neighbours needed for comparison
|
|
@@ -88,14 +80,13 @@ Example 3. - Applying a framework of QC methods (e.g. IntenseQC)
|
|
|
88
80
|
|
|
89
81
|
.. code-block:: python
|
|
90
82
|
|
|
91
|
-
# 3. Applying multiple QC methods in framework (e.g. IntenseQC)
|
|
92
83
|
from rainfallqc.qc_frameworks import apply_qc_framework
|
|
93
84
|
|
|
94
|
-
#
|
|
85
|
+
# 1. Decide which QC methods of IntenseQC will be run
|
|
95
86
|
qc_framework = "IntenseQC"
|
|
96
87
|
qc_methods_to_run = ["QC1", "QC8", "QC9", "QC10", "QC11", "QC12", "QC14", "QC15", "QC16"]
|
|
97
88
|
|
|
98
|
-
#
|
|
89
|
+
# 2 Decide which parameters for QC
|
|
99
90
|
qc_kwargs = {
|
|
100
91
|
"QC1": {"quantile": 5},
|
|
101
92
|
"QC14": {"wet_day_threshold": 1.0, "accumulation_multiplying_factor": 2.0},
|
|
@@ -115,7 +106,7 @@ Example 3. - Applying a framework of QC methods (e.g. IntenseQC)
|
|
|
115
106
|
},
|
|
116
107
|
}
|
|
117
108
|
|
|
118
|
-
# 3.
|
|
109
|
+
# 3. Run QC methods on network data
|
|
119
110
|
qc_result = apply_qc_framework.run_qc_framework(
|
|
120
111
|
daily_gpcc_network, qc_framework=qc_framework, qc_methods_to_run=qc_methods_to_run, qc_kwargs=qc_kwargs
|
|
121
112
|
)
|
|
@@ -68,7 +68,7 @@ homepage = "https://github.com/thomasjkeel/RainfallQC"
|
|
|
68
68
|
target-version = ["py310", "py311", "py312", "py313"]
|
|
69
69
|
|
|
70
70
|
[tool.bumpversion]
|
|
71
|
-
current_version = "0.2.
|
|
71
|
+
current_version = "0.2.5"
|
|
72
72
|
commit = true
|
|
73
73
|
message = "Bump version: {current_version} → {new_version}"
|
|
74
74
|
commit_args = ""
|
|
@@ -21,24 +21,24 @@ import xarray as xr
|
|
|
21
21
|
from rainfallqc.utils import data_utils, neighbourhood_utils
|
|
22
22
|
|
|
23
23
|
MULTIPLYING_FACTORS = {"hourly": 24, "daily": 1} # compared to daily reference
|
|
24
|
-
|
|
24
|
+
GSDR_TIME_RES_CONVERSION = {"1hr": "hourly", "1d": "daily", "1mo": "monthly"}
|
|
25
25
|
GPCC_TIME_RES_CONVERSION = {"tw": "daily", "mw": "monthly"}
|
|
26
|
-
GPCC_HOUR_OFFSET = 7 # Apparently the
|
|
26
|
+
GPCC_HOUR_OFFSET = 7 # Apparently the GSDR data runs from 7am to 7am, so this converts it for comparison
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def
|
|
29
|
+
def read_gsdr_metadata(data_path: str) -> dict:
|
|
30
30
|
"""
|
|
31
|
-
Read the specific format and header of Global Sub-Daily Rainfall (
|
|
31
|
+
Read the specific format and header of Global Sub-Daily Rainfall (GSDR) files.
|
|
32
32
|
|
|
33
33
|
Parameters
|
|
34
34
|
----------
|
|
35
35
|
data_path :
|
|
36
|
-
path to
|
|
36
|
+
path to GSDR data file (.txt)
|
|
37
37
|
|
|
38
38
|
Returns
|
|
39
39
|
-------
|
|
40
40
|
metadata :
|
|
41
|
-
Metadata from
|
|
41
|
+
Metadata from GSDR file
|
|
42
42
|
|
|
43
43
|
"""
|
|
44
44
|
metadata = {}
|
|
@@ -51,7 +51,7 @@ def read_gdsr_metadata(data_path: str) -> dict:
|
|
|
51
51
|
metadata[key] = val
|
|
52
52
|
if key == "other":
|
|
53
53
|
break
|
|
54
|
-
metadata =
|
|
54
|
+
metadata = convert_gsdr_metadata_dates_to_datetime(metadata)
|
|
55
55
|
return metadata
|
|
56
56
|
|
|
57
57
|
|
|
@@ -116,23 +116,23 @@ def read_gpcc_metadata_from_zip(data_path: str, time_res: str, gpcc_file_format:
|
|
|
116
116
|
return gpcc_metadata
|
|
117
117
|
|
|
118
118
|
|
|
119
|
-
def
|
|
119
|
+
def read_gsdr_data_from_file(
|
|
120
120
|
data_path: str,
|
|
121
121
|
raw_data_time_res: str,
|
|
122
122
|
rain_col_prefix: str = None,
|
|
123
123
|
rain_col_suffix: str = None,
|
|
124
124
|
suffix_only: bool = False,
|
|
125
|
-
|
|
125
|
+
gsdr_header_rows: int = 20,
|
|
126
126
|
) -> pl.DataFrame:
|
|
127
127
|
"""
|
|
128
|
-
Read
|
|
128
|
+
Read GSDR data from file.
|
|
129
129
|
|
|
130
|
-
Note: this was developed on the
|
|
130
|
+
Note: this was developed on the GSDR data available from IntenseQC. So it needs a number of header rows in data.
|
|
131
131
|
|
|
132
132
|
Parameters
|
|
133
133
|
----------
|
|
134
134
|
data_path :
|
|
135
|
-
Path to
|
|
135
|
+
Path to GSDR data file
|
|
136
136
|
raw_data_time_res :
|
|
137
137
|
Time resolution of data record i.e. 'hourly' or 'daily'
|
|
138
138
|
rain_col_prefix :
|
|
@@ -141,44 +141,44 @@ def read_gdsr_data_from_file(
|
|
|
141
141
|
Suffix for column name for target_gauge_col (set as None by default)
|
|
142
142
|
suffix_only :
|
|
143
143
|
Override to only include the suffix e.g. if the column name is the ID)
|
|
144
|
-
|
|
144
|
+
gsdr_header_rows :
|
|
145
145
|
Number of rows to skip in the header of the GSDR data (default=20)
|
|
146
146
|
|
|
147
147
|
Returns
|
|
148
148
|
-------
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
gsdr_data :
|
|
150
|
+
GSDR data as Pandas DataFrame
|
|
151
151
|
|
|
152
152
|
"""
|
|
153
153
|
# 1. read in metadata of gauge
|
|
154
|
-
|
|
154
|
+
gsdr_metadata = read_gsdr_metadata(data_path)
|
|
155
155
|
if suffix_only:
|
|
156
156
|
rain_col_name = f"{rain_col_suffix}"
|
|
157
157
|
else:
|
|
158
|
-
rain_col_name = f"{
|
|
158
|
+
rain_col_name = f"{gsdr_metadata['original_units']}"
|
|
159
159
|
if rain_col_prefix:
|
|
160
160
|
rain_col_name = f"{rain_col_prefix}_" + rain_col_name
|
|
161
161
|
if rain_col_suffix:
|
|
162
162
|
rain_col_name += f"_{rain_col_suffix}"
|
|
163
163
|
|
|
164
164
|
# 2. read in gauge data
|
|
165
|
-
|
|
165
|
+
gsdr_data = pl.read_csv(
|
|
166
166
|
data_path,
|
|
167
|
-
skip_rows=
|
|
167
|
+
skip_rows=gsdr_header_rows,
|
|
168
168
|
schema_overrides={rain_col_name: pl.Float64},
|
|
169
169
|
)
|
|
170
170
|
|
|
171
171
|
# 3. add datetime column to data
|
|
172
|
-
|
|
173
|
-
|
|
172
|
+
gsdr_data = add_datetime_to_gsdr_data(
|
|
173
|
+
gsdr_data, gsdr_metadata, multiplying_factor=MULTIPLYING_FACTORS[raw_data_time_res]
|
|
174
174
|
)
|
|
175
|
-
|
|
176
|
-
|
|
175
|
+
gsdr_data = data_utils.replace_missing_vals_with_nan(
|
|
176
|
+
gsdr_data, target_gauge_col=rain_col_name, missing_val=int(gsdr_metadata["no_data_value"])
|
|
177
177
|
)
|
|
178
178
|
|
|
179
179
|
# 4. Select time and rain col
|
|
180
|
-
|
|
181
|
-
return
|
|
180
|
+
gsdr_data = gsdr_data.select(["time", rain_col_name]) # Reorder (to look nice)
|
|
181
|
+
return gsdr_data
|
|
182
182
|
|
|
183
183
|
|
|
184
184
|
def read_gpcc_data_from_zip(
|
|
@@ -260,51 +260,51 @@ def read_gpcc_data_from_zip(
|
|
|
260
260
|
return gpcc_data
|
|
261
261
|
|
|
262
262
|
|
|
263
|
-
def
|
|
263
|
+
def convert_gsdr_metadata_dates_to_datetime(gsdr_metadata: dict) -> dict:
|
|
264
264
|
"""
|
|
265
|
-
Convert
|
|
265
|
+
Convert GSDR metadata date string column to datetime.
|
|
266
266
|
|
|
267
267
|
Parameters
|
|
268
268
|
----------
|
|
269
|
-
|
|
270
|
-
Metadata from
|
|
269
|
+
gsdr_metadata :
|
|
270
|
+
Metadata from GSDR file
|
|
271
271
|
|
|
272
272
|
Returns
|
|
273
273
|
-------
|
|
274
|
-
|
|
275
|
-
Metadata from
|
|
274
|
+
gsdr_metadata : dict
|
|
275
|
+
Metadata from GSDR file with start and end date column
|
|
276
276
|
|
|
277
277
|
"""
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
return
|
|
278
|
+
gsdr_metadata["start_datetime"] = datetime.datetime.strptime(gsdr_metadata["start_datetime"], "%Y%m%d%H")
|
|
279
|
+
gsdr_metadata["end_datetime"] = datetime.datetime.strptime(gsdr_metadata["end_datetime"], "%Y%m%d%H")
|
|
280
|
+
return gsdr_metadata
|
|
281
281
|
|
|
282
282
|
|
|
283
|
-
def
|
|
284
|
-
|
|
283
|
+
def add_datetime_to_gsdr_data(
|
|
284
|
+
gsdr_data: pl.DataFrame, gsdr_metadata: dict, multiplying_factor: int | float
|
|
285
285
|
) -> pl.DataFrame:
|
|
286
286
|
"""
|
|
287
|
-
Add datetime column to
|
|
287
|
+
Add datetime column to GSDR gauge data using metadata from that gauge.
|
|
288
288
|
|
|
289
289
|
NOTE: Could maybe extend so can find metadata if not provided?
|
|
290
290
|
|
|
291
291
|
Parameters
|
|
292
292
|
----------
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
Metadata from
|
|
293
|
+
gsdr_data :
|
|
294
|
+
GSDR data
|
|
295
|
+
gsdr_metadata :
|
|
296
|
+
Metadata from GSDR file
|
|
297
297
|
multiplying_factor : int or float
|
|
298
298
|
Factor to multiply the data by.
|
|
299
299
|
|
|
300
300
|
Returns
|
|
301
301
|
-------
|
|
302
|
-
|
|
303
|
-
|
|
302
|
+
gsdr_data
|
|
303
|
+
GSDR data with datetime column added
|
|
304
304
|
|
|
305
305
|
"""
|
|
306
|
-
start_date =
|
|
307
|
-
end_date =
|
|
306
|
+
start_date = gsdr_metadata["start_datetime"]
|
|
307
|
+
end_date = gsdr_metadata["end_datetime"]
|
|
308
308
|
assert isinstance(start_date, datetime.datetime), (
|
|
309
309
|
"Please convert start_ and end_datetime to datetime.datetime objects"
|
|
310
310
|
)
|
|
@@ -315,10 +315,10 @@ def add_datetime_to_gdsr_data(
|
|
|
315
315
|
date_interval.append(start_date + datetime.timedelta(hours=i))
|
|
316
316
|
|
|
317
317
|
# add time column
|
|
318
|
-
assert len(
|
|
319
|
-
|
|
318
|
+
assert len(gsdr_data) == len(date_interval)
|
|
319
|
+
gsdr_data = gsdr_data.with_columns(time=pl.Series(date_interval))
|
|
320
320
|
|
|
321
|
-
return
|
|
321
|
+
return gsdr_data
|
|
322
322
|
|
|
323
323
|
|
|
324
324
|
def convert_data_hourly_to_daily(hourly_data: pl.DataFrame, rain_cols: List[str], hour_offset: int) -> pl.DataFrame:
|
|
@@ -383,32 +383,32 @@ def load_etccdi_data(etccdi_var: str, path_to_etccdi: str = None) -> xr.Dataset:
|
|
|
383
383
|
return etccdi_data
|
|
384
384
|
|
|
385
385
|
|
|
386
|
-
def
|
|
386
|
+
def load_gsdr_gauge_network_metadata(path_to_gsdr_dir: str, file_format: str = ".txt") -> pl.DataFrame:
|
|
387
387
|
"""
|
|
388
|
-
Load metadata from
|
|
388
|
+
Load metadata from GSDR gauges from a directory.
|
|
389
389
|
|
|
390
390
|
Parameters
|
|
391
391
|
----------
|
|
392
|
-
|
|
393
|
-
Path to directory with
|
|
392
|
+
path_to_gsdr_dir :
|
|
393
|
+
Path to directory with GSDR gauges
|
|
394
394
|
file_format :
|
|
395
395
|
Format of file (default is .txt)
|
|
396
396
|
|
|
397
397
|
Returns
|
|
398
398
|
-------
|
|
399
399
|
all_station_metadata :
|
|
400
|
-
All
|
|
400
|
+
All GSDR gauges metadata as one dataframe.
|
|
401
401
|
|
|
402
402
|
"""
|
|
403
403
|
# 1. Glob all metadata paths
|
|
404
|
-
if not os.path.isdir(
|
|
405
|
-
raise ValueError(f"Invalid
|
|
406
|
-
all_metadata_data_paths = glob.glob(f"{
|
|
404
|
+
if not os.path.isdir(path_to_gsdr_dir):
|
|
405
|
+
raise ValueError(f"Invalid GSDR metadata directory at {path_to_gsdr_dir}")
|
|
406
|
+
all_metadata_data_paths = glob.glob(f"{path_to_gsdr_dir}*{file_format}")
|
|
407
407
|
|
|
408
|
-
# 2. Load all
|
|
408
|
+
# 2. Load all GSDR metadata from data
|
|
409
409
|
all_station_metadata_list = []
|
|
410
410
|
for file in all_metadata_data_paths:
|
|
411
|
-
one_station_metadata =
|
|
411
|
+
one_station_metadata = read_gsdr_metadata(data_path=file)
|
|
412
412
|
all_station_metadata_list.append(one_station_metadata)
|
|
413
413
|
|
|
414
414
|
# 3. Convert to pl.DataFrame
|
|
@@ -548,43 +548,43 @@ class GaugeNetworkReader(ABC):
|
|
|
548
548
|
return all_neighbour_ids
|
|
549
549
|
|
|
550
550
|
|
|
551
|
-
class
|
|
552
|
-
"""
|
|
551
|
+
class GSDRNetworkReader(GaugeNetworkReader):
|
|
552
|
+
"""GSDR rain gauge network reader."""
|
|
553
553
|
|
|
554
|
-
def __init__(self,
|
|
554
|
+
def __init__(self, path_to_gsdr_dir: str, file_format: str = ".txt"):
|
|
555
555
|
"""Load network reader."""
|
|
556
|
-
self.
|
|
556
|
+
self.path_to_gsdr_dir = path_to_gsdr_dir
|
|
557
557
|
self.file_format = file_format
|
|
558
|
-
super().__init__(
|
|
558
|
+
super().__init__(path_to_gsdr_dir)
|
|
559
559
|
self.data_paths = self._get_data_paths()
|
|
560
560
|
self.metadata = self._add_paths_to_metadata()
|
|
561
561
|
self.time_res = self.metadata["new_timestep"][0]
|
|
562
562
|
|
|
563
563
|
def _load_metadata(self) -> pl.DataFrame:
|
|
564
564
|
"""
|
|
565
|
-
Load metadata from
|
|
565
|
+
Load metadata from GSDR gauge metadata path.
|
|
566
566
|
|
|
567
567
|
Returns
|
|
568
568
|
-------
|
|
569
569
|
metadata :
|
|
570
|
-
Metadata of
|
|
570
|
+
Metadata of GSDR gauges.
|
|
571
571
|
|
|
572
572
|
"""
|
|
573
|
-
metadata =
|
|
573
|
+
metadata = load_gsdr_gauge_network_metadata(self.path_to_gsdr_dir, self.file_format)
|
|
574
574
|
return metadata
|
|
575
575
|
|
|
576
576
|
def _get_data_paths(self) -> dict:
|
|
577
577
|
"""
|
|
578
|
-
Get paths to gauge network of
|
|
578
|
+
Get paths to gauge network of GSDR gauges.
|
|
579
579
|
|
|
580
580
|
Returns
|
|
581
581
|
-------
|
|
582
582
|
gauge_paths :
|
|
583
|
-
Dataframe of all
|
|
583
|
+
Dataframe of all GSDR gauges rain record.
|
|
584
584
|
|
|
585
585
|
"""
|
|
586
586
|
gauge_paths = get_paths_using_gauge_ids(
|
|
587
|
-
self.metadata["station_id"], self.
|
|
587
|
+
self.metadata["station_id"], self.path_to_gsdr_dir, file_format=self.file_format
|
|
588
588
|
)
|
|
589
589
|
return gauge_paths
|
|
590
590
|
|
|
@@ -598,10 +598,10 @@ class GDSRNetworkReader(GaugeNetworkReader):
|
|
|
598
598
|
rain_col_prefix: str,
|
|
599
599
|
data_paths: List[str] | np.ndarray[str],
|
|
600
600
|
suffix_only: bool = False,
|
|
601
|
-
|
|
601
|
+
gsdr_header_rows: int = 20,
|
|
602
602
|
) -> pl.DataFrame:
|
|
603
603
|
"""
|
|
604
|
-
Load
|
|
604
|
+
Load GSDR network data based on file paths.
|
|
605
605
|
|
|
606
606
|
Parameters
|
|
607
607
|
----------
|
|
@@ -611,28 +611,28 @@ class GDSRNetworkReader(GaugeNetworkReader):
|
|
|
611
611
|
Prefix for rain column name (default is 'rain')
|
|
612
612
|
suffix_only :
|
|
613
613
|
Override to only include the suffix e.g. if the column name is the ID)
|
|
614
|
-
|
|
614
|
+
gsdr_header_rows :
|
|
615
615
|
Number of rows to skip in the header of the GSDR data (default=20)
|
|
616
616
|
|
|
617
617
|
Returns
|
|
618
618
|
-------
|
|
619
619
|
network_data :
|
|
620
|
-
Dataframe of
|
|
620
|
+
Dataframe of GSDR gauges.
|
|
621
621
|
|
|
622
622
|
"""
|
|
623
623
|
for ind, path in enumerate(data_paths):
|
|
624
624
|
# 1. get gauge_id name
|
|
625
|
-
|
|
626
|
-
|
|
625
|
+
gsdr_file_name = path.split("/")[-1]
|
|
626
|
+
gsdr_name = gsdr_file_name.split(".")[0]
|
|
627
627
|
|
|
628
628
|
# 2. Read in one gauge
|
|
629
|
-
one_gauge =
|
|
629
|
+
one_gauge = read_gsdr_data_from_file(
|
|
630
630
|
data_path=path,
|
|
631
|
-
raw_data_time_res=
|
|
631
|
+
raw_data_time_res=GSDR_TIME_RES_CONVERSION[self.time_res],
|
|
632
632
|
rain_col_prefix=rain_col_prefix,
|
|
633
|
-
rain_col_suffix=
|
|
633
|
+
rain_col_suffix=gsdr_name,
|
|
634
634
|
suffix_only=suffix_only,
|
|
635
|
-
|
|
635
|
+
gsdr_header_rows=gsdr_header_rows,
|
|
636
636
|
)
|
|
637
637
|
|
|
638
638
|
# 3. Join data together
|
|
@@ -679,7 +679,7 @@ class GPCCNetworkReader(GaugeNetworkReader):
|
|
|
679
679
|
Returns
|
|
680
680
|
-------
|
|
681
681
|
gauge_paths :
|
|
682
|
-
Dataframe of all
|
|
682
|
+
Dataframe of all GSDR gauges rain record.
|
|
683
683
|
|
|
684
684
|
"""
|
|
685
685
|
gauge_paths = get_paths_using_gauge_ids(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|