climpact 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- climpact/__init__.py +6 -0
- climpact/batch.py +470 -0
- climpact/calendars.py +224 -0
- climpact/cli.py +786 -0
- climpact/compare.py +166 -0
- climpact/config.py +76 -0
- climpact/data/__init__.py +1 -0
- climpact/data/climate.indices.csv +89 -0
- climpact/gridded/__init__.py +22 -0
- climpact/gridded/drought.py +291 -0
- climpact/gridded/heatwaves.py +196 -0
- climpact/gridded/indices.py +951 -0
- climpact/gridded/io.py +626 -0
- climpact/gridded/thresholds.py +488 -0
- climpact/io/__init__.py +1 -0
- climpact/io/climpact_csv.py +287 -0
- climpact/io/output_files.py +334 -0
- climpact/io/station.py +151 -0
- climpact/missing.py +125 -0
- climpact/modules.py +497 -0
- climpact/outputs.py +126 -0
- climpact/periods.py +147 -0
- climpact/plots.py +601 -0
- climpact/products/__init__.py +3 -0
- climpact/products/drought.py +450 -0
- climpact/products/ehf.py +230 -0
- climpact/products/heatwaves.py +268 -0
- climpact/products/indices.py +1049 -0
- climpact/products/qc.py +160 -0
- climpact/products/tables.py +66 -0
- climpact/products/thresholds.py +65 -0
- climpact/products/trends.py +214 -0
- climpact/progress.py +62 -0
- climpact/qc.py +244 -0
- climpact/rdist.py +1278 -0
- climpact/report.py +792 -0
- climpact/rnumerics.py +327 -0
- climpact/sector.py +749 -0
- climpact/station.py +69 -0
- climpact/thresholds.py +482 -0
- climpact/trends.py +285 -0
- climpact-0.1.0.dist-info/METADATA +63 -0
- climpact-0.1.0.dist-info/RECORD +46 -0
- climpact-0.1.0.dist-info/WHEEL +4 -0
- climpact-0.1.0.dist-info/entry_points.txt +2 -0
- climpact-0.1.0.dist-info/licenses/LICENSE +675 -0
climpact/__init__.py
ADDED
climpact/batch.py
ADDED
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
"""Batch processing: many stations from one metadata file.
|
|
2
|
+
|
|
3
|
+
Ports Climpact's batch mode (``climpact.batch.stations.r`` and ``server/batch_stations.R``).
|
|
4
|
+
The metadata file is R's, read unchanged: a header row, then one row per station with 12
|
|
5
|
+
whitespace-separated columns::
|
|
6
|
+
|
|
7
|
+
station_file latitude longitude wsdin csdin Tb_HDD Tb_CDD Tb_GDD rxnday rnnmm txtn SPEI
|
|
8
|
+
|
|
9
|
+
``station_file`` is a file name inside the data folder. The station name is the file name
|
|
10
|
+
without its last extension, as R's ``strip_file_extension``. The base period applies to
|
|
11
|
+
every station. As in R, batch runs calculate no custom index unless one is requested.
|
|
12
|
+
|
|
13
|
+
Each station is run in its own process and written to ``<out>/<station>/`` exactly as
|
|
14
|
+
``climpact run`` writes it. A station that fails is reported in ``<out>/<station>.errors.txt``
|
|
15
|
+
(R's error log name) and the rest of the batch carries on.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import datetime as dt
|
|
21
|
+
import multiprocessing
|
|
22
|
+
import traceback
|
|
23
|
+
from collections.abc import Callable, Iterable
|
|
24
|
+
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait
|
|
25
|
+
from dataclasses import dataclass, replace
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from queue import Empty
|
|
28
|
+
|
|
29
|
+
from climpact.config import CustomIndex, IndexParams, QCParams, StationMetadata
|
|
30
|
+
from climpact.modules import ModuleSelection
|
|
31
|
+
from climpact.progress import BatchProgress, Progress, Steps
|
|
32
|
+
|
|
33
|
+
# R's read.file.list.metadata col.names. The header row's own names are not checked.
|
|
34
|
+
COLUMNS = (
|
|
35
|
+
"station_file",
|
|
36
|
+
"latitude",
|
|
37
|
+
"longitude",
|
|
38
|
+
"wsdin",
|
|
39
|
+
"csdin",
|
|
40
|
+
"Tb_HDD",
|
|
41
|
+
"Tb_CDD",
|
|
42
|
+
"Tb_GDD",
|
|
43
|
+
"rxnday",
|
|
44
|
+
"rnnmm",
|
|
45
|
+
"txtn",
|
|
46
|
+
"SPEI",
|
|
47
|
+
)
|
|
48
|
+
_INTEGER_COLUMNS = {"wsdin", "csdin", "rxnday", "txtn", "SPEI"}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class BatchMetadataError(ValueError):
|
|
52
|
+
"""The metadata file cannot be read; R's ``read.table`` would stop the whole batch."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class BatchStation:
|
|
57
|
+
"""One row of the batch metadata file."""
|
|
58
|
+
|
|
59
|
+
station_file: str
|
|
60
|
+
latitude: float
|
|
61
|
+
longitude: float
|
|
62
|
+
params: IndexParams
|
|
63
|
+
line: int # line number in the metadata file, for messages
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def name(self) -> str:
|
|
67
|
+
return strip_file_extension(self.station_file)
|
|
68
|
+
|
|
69
|
+
def metadata(self, base_start: int, base_end: int) -> StationMetadata:
|
|
70
|
+
return StationMetadata(
|
|
71
|
+
name=self.name,
|
|
72
|
+
latitude=self.latitude,
|
|
73
|
+
longitude=self.longitude,
|
|
74
|
+
base_start=base_start,
|
|
75
|
+
base_end=base_end,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class StationResult:
|
|
81
|
+
"""Outcome of one station: the files written, or the error that stopped it."""
|
|
82
|
+
|
|
83
|
+
name: str
|
|
84
|
+
station_file: str
|
|
85
|
+
written: int = 0
|
|
86
|
+
skipped: tuple[str, ...] = ()
|
|
87
|
+
error: str | None = None
|
|
88
|
+
error_file: Path | None = None
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def ok(self) -> bool:
|
|
92
|
+
return self.error is None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def strip_file_extension(file_name: str) -> str:
|
|
96
|
+
"""File name without its last extension (``a.b.txt`` -> ``a.b``), as R's function."""
|
|
97
|
+
stem, dot, _ = file_name.rpartition(".")
|
|
98
|
+
return stem if dot else file_name
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _number(text: str, column: str, line: int) -> float | int:
|
|
102
|
+
try:
|
|
103
|
+
value = float(text)
|
|
104
|
+
except ValueError:
|
|
105
|
+
raise BatchMetadataError(f"line {line}: {column} must be a number, got {text!r}") from None
|
|
106
|
+
if column in _INTEGER_COLUMNS:
|
|
107
|
+
if not value.is_integer():
|
|
108
|
+
raise BatchMetadataError(f"line {line}: {column} must be a whole number, got {text!r}")
|
|
109
|
+
return int(value)
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def validate_params(params: IndexParams) -> None:
|
|
114
|
+
"""Raise ``ValueError`` for parameter values R's ``validate_climdexInputParams`` rejects."""
|
|
115
|
+
problems = []
|
|
116
|
+
if not 1 <= params.wsdi_days <= 10:
|
|
117
|
+
problems.append("WSDId: value must be between 1 and 10")
|
|
118
|
+
if not 1 <= params.csdi_days <= 10:
|
|
119
|
+
problems.append("CSDId value must be between 1 and 10")
|
|
120
|
+
if params.rx_days < 1:
|
|
121
|
+
problems.append("RXnDAY: value must be a positive number")
|
|
122
|
+
if params.txtn_days < 1:
|
|
123
|
+
problems.append("TXdTNd and TXbdTNbd: value must be a positive number")
|
|
124
|
+
if params.rnnmm < 0:
|
|
125
|
+
problems.append("Rnnmm: value must be greater than or equal to zero")
|
|
126
|
+
if params.spei_months < 1:
|
|
127
|
+
problems.append("Custom SPEI/SPI time scale value must be a positive number")
|
|
128
|
+
if problems:
|
|
129
|
+
raise ValueError("\n".join(problems))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def read_batch_metadata(path: str | Path) -> list[BatchStation]:
|
|
133
|
+
"""Read R's batch metadata file (``read.file.list.metadata``).
|
|
134
|
+
|
|
135
|
+
As ``read.table(header = TRUE)``: fields are separated by any whitespace, the first
|
|
136
|
+
non-blank line is the header, blank lines and ``#`` comments are ignored. A row with
|
|
137
|
+
the wrong number of fields or a non-numeric value is an error for the whole file.
|
|
138
|
+
"""
|
|
139
|
+
rows = []
|
|
140
|
+
for number, raw in enumerate(Path(path).read_text().splitlines(), start=1):
|
|
141
|
+
fields = raw.split("#", 1)[0].split()
|
|
142
|
+
if fields:
|
|
143
|
+
rows.append((number, fields))
|
|
144
|
+
if not rows:
|
|
145
|
+
raise BatchMetadataError(f"{path}: no header row")
|
|
146
|
+
stations = []
|
|
147
|
+
for number, fields in rows[1:]:
|
|
148
|
+
if len(fields) != len(COLUMNS):
|
|
149
|
+
raise BatchMetadataError(
|
|
150
|
+
f"{path}, line {number}: {len(fields)} fields, expected {len(COLUMNS)} "
|
|
151
|
+
f"({' '.join(COLUMNS)})"
|
|
152
|
+
)
|
|
153
|
+
values = dict(zip(COLUMNS, fields, strict=True))
|
|
154
|
+
n = {c: _number(values[c], c, number) for c in COLUMNS[1:]}
|
|
155
|
+
params = IndexParams(
|
|
156
|
+
wsdi_days=n["wsdin"],
|
|
157
|
+
csdi_days=n["csdin"],
|
|
158
|
+
rx_days=n["rxnday"],
|
|
159
|
+
txtn_days=n["txtn"],
|
|
160
|
+
hdd_base=n["Tb_HDD"],
|
|
161
|
+
cdd_base=n["Tb_CDD"],
|
|
162
|
+
gdd_base=n["Tb_GDD"],
|
|
163
|
+
rnnmm=n["rnnmm"],
|
|
164
|
+
spei_months=n["SPEI"],
|
|
165
|
+
custom=None,
|
|
166
|
+
)
|
|
167
|
+
stations.append(
|
|
168
|
+
BatchStation(values["station_file"], n["latitude"], n["longitude"], params, number)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
seen: dict[str, BatchStation] = {}
|
|
172
|
+
for station in stations:
|
|
173
|
+
if station.name in seen:
|
|
174
|
+
raise BatchMetadataError(
|
|
175
|
+
f"{path}, lines {seen[station.name].line} and {station.line}: "
|
|
176
|
+
f"{seen[station.name].station_file} and {station.station_file} "
|
|
177
|
+
f"would both write to the station folder {station.name!r}"
|
|
178
|
+
)
|
|
179
|
+
seen[station.name] = station
|
|
180
|
+
return stations
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def run_station(
|
|
184
|
+
station_file: str | Path,
|
|
185
|
+
metadata: StationMetadata,
|
|
186
|
+
params: IndexParams,
|
|
187
|
+
out_dir: str | Path,
|
|
188
|
+
date: dt.date | None = None,
|
|
189
|
+
modules: ModuleSelection | None = None,
|
|
190
|
+
progress: Progress | None = None,
|
|
191
|
+
plots: bool = False,
|
|
192
|
+
zip: bool = False,
|
|
193
|
+
qc: QCParams | None = None,
|
|
194
|
+
plot_title: str | None = None,
|
|
195
|
+
report: bool = False,
|
|
196
|
+
report_path: str | Path | None = None,
|
|
197
|
+
) -> tuple[dict[str, Path], list[str]]:
|
|
198
|
+
"""Build and write every output for one station, as ``climpact run`` does.
|
|
199
|
+
|
|
200
|
+
``modules`` selects the QC, homogeneity and index modules (default: the built-in
|
|
201
|
+
ones). ``plots`` also draws R's index plots and combined PDF
|
|
202
|
+
(:func:`climpact.plots.write_plots`, needs matplotlib), and ``zip`` then zips the
|
|
203
|
+
station folder to ``<out_dir>/<station>.zip`` (:func:`climpact.plots.zip_station`).
|
|
204
|
+
``report`` also writes the PDF report (:func:`climpact.report.write_report`, needs
|
|
205
|
+
matplotlib) to ``report_path``, by default ``<out_dir>/<station>_report.pdf``, beside the
|
|
206
|
+
station folder so the zip keeps R's file list.
|
|
207
|
+
``plot_title`` replaces the station title at the top of every plot;
|
|
208
|
+
``qc`` holds the quality-control thresholds (default: the R app's, :class:`QCParams`).
|
|
209
|
+
With no index modules (``ModuleSelection(indices=())``) only the QC files are written,
|
|
210
|
+
as the R app's quality-control step does.
|
|
211
|
+
``progress(stage, step, total)`` is called after each stage: ``read``, the stages of
|
|
212
|
+
:func:`~climpact.modules.pipeline_stages`, ``write``, then with ``plots`` the stages of
|
|
213
|
+
:func:`~climpact.plots.plot_steps`, with ``report`` a ``report`` and with ``zip`` a last
|
|
214
|
+
``zip`` (see :mod:`climpact.progress`). Returns the files written, keyed by output key
|
|
215
|
+
(plots as ``plots/<name>``, the report as ``report``, the zip file as ``zip``), and the
|
|
216
|
+
output keys not written (missing in every period).
|
|
217
|
+
"""
|
|
218
|
+
from climpact.io.output_files import write_outputs
|
|
219
|
+
from climpact.modules import pipeline_stages, resolve, run_pipeline
|
|
220
|
+
from climpact.station import load_station
|
|
221
|
+
|
|
222
|
+
if plots or report:
|
|
223
|
+
from climpact.plots import plot_steps, require_matplotlib
|
|
224
|
+
|
|
225
|
+
require_matplotlib() # fail before the run, not after it
|
|
226
|
+
loaded = resolve(modules)
|
|
227
|
+
steps = Steps(progress, 0)
|
|
228
|
+
pipeline_progress = None
|
|
229
|
+
if progress is not None:
|
|
230
|
+
steps.total = len(pipeline_stages(loaded)) + 2
|
|
231
|
+
steps.total += len(plot_steps(params)) if plots else 0
|
|
232
|
+
steps.total += 1 if report else 0
|
|
233
|
+
steps.total += 1 if zip else 0
|
|
234
|
+
|
|
235
|
+
def pipeline_progress(stage: str, step: int, _total: int) -> None:
|
|
236
|
+
steps.done(stage)
|
|
237
|
+
|
|
238
|
+
run = load_station(station_file, metadata, params, qc)
|
|
239
|
+
steps.done("read")
|
|
240
|
+
tables = run_pipeline(run, loaded, pipeline_progress).tables
|
|
241
|
+
written = write_outputs(tables, metadata, params, out_dir, date)
|
|
242
|
+
skipped = sorted(set(tables) - set(written))
|
|
243
|
+
steps.done("write")
|
|
244
|
+
if plots:
|
|
245
|
+
from climpact.plots import write_plots
|
|
246
|
+
|
|
247
|
+
drawn = write_plots(tables, metadata, params, out_dir, steps.done, title=plot_title)
|
|
248
|
+
written.update({f"plots/{name}": path for name, path in drawn.items()})
|
|
249
|
+
if report:
|
|
250
|
+
from climpact.report import write_report
|
|
251
|
+
|
|
252
|
+
written["report"] = write_report(
|
|
253
|
+
out_dir, metadata, params, report_path, qc=qc, title=plot_title, date=date
|
|
254
|
+
)
|
|
255
|
+
steps.done("report")
|
|
256
|
+
if zip:
|
|
257
|
+
from climpact.plots import zip_station
|
|
258
|
+
|
|
259
|
+
written["zip"] = zip_station(out_dir, metadata.name)
|
|
260
|
+
steps.done("zip")
|
|
261
|
+
return written, skipped
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _process(
|
|
265
|
+
station: BatchStation,
|
|
266
|
+
data_dir: Path,
|
|
267
|
+
out_dir: Path,
|
|
268
|
+
base_start: int,
|
|
269
|
+
base_end: int,
|
|
270
|
+
date: dt.date | None,
|
|
271
|
+
modules: ModuleSelection | None,
|
|
272
|
+
plots: bool = False,
|
|
273
|
+
report_dir: Path | None = None,
|
|
274
|
+
progress: BatchProgress | None = None,
|
|
275
|
+
) -> StationResult:
|
|
276
|
+
"""Run one station, turning any exception into a failed result."""
|
|
277
|
+
result = StationResult(station.name, station.station_file)
|
|
278
|
+
station_progress = None
|
|
279
|
+
if progress is not None:
|
|
280
|
+
|
|
281
|
+
def station_progress(stage: str, step: int, total: int) -> None:
|
|
282
|
+
progress(station.name, stage, step, total)
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
from climpact.report import report_file
|
|
286
|
+
|
|
287
|
+
validate_params(station.params)
|
|
288
|
+
path = data_dir / station.station_file
|
|
289
|
+
if not path.is_file():
|
|
290
|
+
raise FileNotFoundError(f"station file not found: {path}")
|
|
291
|
+
written, skipped = run_station(
|
|
292
|
+
path,
|
|
293
|
+
station.metadata(base_start, base_end),
|
|
294
|
+
station.params,
|
|
295
|
+
out_dir,
|
|
296
|
+
date,
|
|
297
|
+
modules,
|
|
298
|
+
station_progress,
|
|
299
|
+
plots=plots,
|
|
300
|
+
report=report_dir is not None,
|
|
301
|
+
report_path=None if report_dir is None else report_file(report_dir, station.name),
|
|
302
|
+
)
|
|
303
|
+
except Exception as exc: # noqa: BLE001 - reported per station, the batch carries on
|
|
304
|
+
detail = "".join(traceback.format_exception_only(exc)).strip()
|
|
305
|
+
return replace(result, error=detail)
|
|
306
|
+
return replace(result, written=len(written), skipped=tuple(skipped))
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# Seconds to wait for the end of a finished station's progress, which is already sent.
|
|
310
|
+
_QUEUE_TIMEOUT = 60
|
|
311
|
+
|
|
312
|
+
# The progress queue of a worker process (set by _init_worker), or None.
|
|
313
|
+
_QUEUE = None
|
|
314
|
+
# The stage that marks the end of a station's progress on the queue.
|
|
315
|
+
_END = None
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _init_worker(queue) -> None:
|
|
319
|
+
global _QUEUE
|
|
320
|
+
_QUEUE = queue
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _send(station: str, stage: str | None, step: int, total: int) -> None:
|
|
324
|
+
_QUEUE.put((station, stage, step, total))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _process_in_worker(station: BatchStation, *args) -> StationResult:
|
|
328
|
+
"""``_process`` in a worker process, sending its progress to the parent's queue."""
|
|
329
|
+
if _QUEUE is None:
|
|
330
|
+
return _process(station, *args)
|
|
331
|
+
try:
|
|
332
|
+
return _process(station, *args, progress=_send)
|
|
333
|
+
finally:
|
|
334
|
+
_send(station.name, _END, 0, 0)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def run_batch(
|
|
338
|
+
stations: Iterable[BatchStation],
|
|
339
|
+
data_dir: str | Path,
|
|
340
|
+
out_dir: str | Path,
|
|
341
|
+
base_start: int = 1971,
|
|
342
|
+
base_end: int = 2000,
|
|
343
|
+
custom: CustomIndex | None = None,
|
|
344
|
+
workers: int | None = None,
|
|
345
|
+
date: dt.date | None = None,
|
|
346
|
+
on_result: Callable[[StationResult], None] | None = None,
|
|
347
|
+
modules: ModuleSelection | None = None,
|
|
348
|
+
progress: BatchProgress | None = None,
|
|
349
|
+
plots: bool = False,
|
|
350
|
+
zip: bool = False,
|
|
351
|
+
report_dir: str | Path | None = None,
|
|
352
|
+
) -> list[StationResult]:
|
|
353
|
+
"""Run every station and write its outputs to ``<out_dir>/<station>/``.
|
|
354
|
+
|
|
355
|
+
Stations run in a pool of ``workers`` processes (default: one per CPU, at most one
|
|
356
|
+
per station); ``workers=1`` runs them in this process. A station that fails does not
|
|
357
|
+
stop the others: its error is written to ``<out_dir>/<station>.errors.txt``, and an
|
|
358
|
+
error file left by an earlier run is removed first, as R does. ``custom`` is the
|
|
359
|
+
custom index calculated for every station (R calculates none in batch mode).
|
|
360
|
+
``modules`` selects the QC, homogeneity and index modules and their options
|
|
361
|
+
(default: the built-in ones); they are checked here, so an unknown module or a bad
|
|
362
|
+
option raises before any station runs, and each worker process then loads them
|
|
363
|
+
itself.
|
|
364
|
+
``on_result`` is called as each station finishes. Results are returned in metadata
|
|
365
|
+
order.
|
|
366
|
+
|
|
367
|
+
``progress(station, stage, step, total)`` is called after each stage of each
|
|
368
|
+
station, as :func:`run_station` reports them (see :mod:`climpact.progress`). With
|
|
369
|
+
several workers the stages are sent back from the worker processes on a queue and the
|
|
370
|
+
callback runs in this process and thread; a station's stages always come before its
|
|
371
|
+
``on_result``, and stages of different stations interleave.
|
|
372
|
+
|
|
373
|
+
``plots`` draws each station's plots and combined PDF, as :func:`run_station` does.
|
|
374
|
+
``zip`` zips the whole batch folder once every station has finished, to
|
|
375
|
+
``<out_dir>.zip`` next to it (:func:`climpact.plots.zip_batch`), as R's batch
|
|
376
|
+
download does. ``report_dir`` writes each station's PDF report
|
|
377
|
+
(:func:`climpact.report.write_report`) to ``<report_dir>/<station>_report.pdf``; it
|
|
378
|
+
should be outside ``out_dir``, which is zipped as R's batch folder.
|
|
379
|
+
"""
|
|
380
|
+
from climpact.modules import resolve
|
|
381
|
+
|
|
382
|
+
resolve(modules) # fail before the batch, not in every station
|
|
383
|
+
if plots or report_dir is not None:
|
|
384
|
+
from climpact.plots import require_matplotlib
|
|
385
|
+
|
|
386
|
+
require_matplotlib() # fail before the batch, not in every station
|
|
387
|
+
stations = [replace(s, params=replace(s.params, custom=custom)) for s in stations]
|
|
388
|
+
data_dir, out_dir = Path(data_dir), Path(out_dir)
|
|
389
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
390
|
+
for station in stations:
|
|
391
|
+
_error_file(out_dir, station.name).unlink(missing_ok=True)
|
|
392
|
+
|
|
393
|
+
report_dir = Path(report_dir) if report_dir is not None else None
|
|
394
|
+
args = [
|
|
395
|
+
(s, data_dir, out_dir, base_start, base_end, date, modules, plots, report_dir)
|
|
396
|
+
for s in stations
|
|
397
|
+
]
|
|
398
|
+
workers = min(workers or multiprocessing.cpu_count(), max(len(stations), 1))
|
|
399
|
+
results: dict[str, StationResult] = {}
|
|
400
|
+
|
|
401
|
+
def finish(result: StationResult) -> None:
|
|
402
|
+
if not result.ok:
|
|
403
|
+
path = _error_file(out_dir, result.name)
|
|
404
|
+
path.write_text(result.error + "\n")
|
|
405
|
+
result = replace(result, error_file=path)
|
|
406
|
+
results[result.name] = result
|
|
407
|
+
if on_result:
|
|
408
|
+
on_result(result)
|
|
409
|
+
|
|
410
|
+
if workers <= 1:
|
|
411
|
+
for a in args:
|
|
412
|
+
finish(_process(*a, progress=progress))
|
|
413
|
+
return _batch_done(stations, results, out_dir, zip)
|
|
414
|
+
|
|
415
|
+
# spawn: the same start method on every platform, and no fork of a process whose
|
|
416
|
+
# numerical libraries may be running threads.
|
|
417
|
+
context = multiprocessing.get_context("spawn")
|
|
418
|
+
# Progress comes back on a queue given to each worker when it starts. Each station's
|
|
419
|
+
# stages end with an _END marker, and its result is handled only once the marker has
|
|
420
|
+
# arrived, so its stages reach ``progress`` before its ``on_result``.
|
|
421
|
+
queue = context.Queue() if progress is not None else None
|
|
422
|
+
ended: set[str] = set()
|
|
423
|
+
|
|
424
|
+
def receive(item: tuple) -> None:
|
|
425
|
+
if item[1] is _END:
|
|
426
|
+
ended.add(item[0])
|
|
427
|
+
else:
|
|
428
|
+
progress(*item)
|
|
429
|
+
|
|
430
|
+
with ProcessPoolExecutor(
|
|
431
|
+
max_workers=workers, mp_context=context, initializer=_init_worker, initargs=(queue,)
|
|
432
|
+
) as pool:
|
|
433
|
+
futures = {pool.submit(_process_in_worker, *a): a[0] for a in args}
|
|
434
|
+
pending = set(futures)
|
|
435
|
+
while pending:
|
|
436
|
+
done, pending = wait(
|
|
437
|
+
pending, timeout=0.1 if queue else None, return_when=FIRST_COMPLETED
|
|
438
|
+
)
|
|
439
|
+
while queue is not None and not queue.empty():
|
|
440
|
+
receive(queue.get())
|
|
441
|
+
for future in done:
|
|
442
|
+
station = futures[future]
|
|
443
|
+
try:
|
|
444
|
+
result = future.result()
|
|
445
|
+
except Exception as exc: # noqa: BLE001 - e.g. a worker killed by the OS
|
|
446
|
+
detail = "".join(traceback.format_exception_only(exc)).strip()
|
|
447
|
+
result = StationResult(station.name, station.station_file, error=detail)
|
|
448
|
+
else:
|
|
449
|
+
# The worker sent the marker before returning; wait for it to arrive.
|
|
450
|
+
try:
|
|
451
|
+
while queue is not None and station.name not in ended:
|
|
452
|
+
receive(queue.get(timeout=_QUEUE_TIMEOUT))
|
|
453
|
+
except Empty: # never expected: the marker was sent before returning
|
|
454
|
+
pass
|
|
455
|
+
finish(result)
|
|
456
|
+
return _batch_done(stations, results, out_dir, zip)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _batch_done(
|
|
460
|
+
stations: list[BatchStation], results: dict[str, StationResult], out_dir: Path, zip: bool
|
|
461
|
+
) -> list[StationResult]:
|
|
462
|
+
if zip:
|
|
463
|
+
from climpact.plots import zip_batch
|
|
464
|
+
|
|
465
|
+
zip_batch(out_dir)
|
|
466
|
+
return [results[s.name] for s in stations]
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _error_file(out_dir: Path, name: str) -> Path:
|
|
470
|
+
return out_dir / f"{name}.errors.txt"
|
climpact/calendars.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Daily time axes in the CF calendars without leap years: ``noleap`` (``365_day``)
|
|
2
|
+
and ``360_day``.
|
|
3
|
+
|
|
4
|
+
Climate model output often uses them. Climpact's gridded code reads them as PCICt
|
|
5
|
+
dates, and climdex.pcic then works with ``dpy`` days per year (``attr(ts, "dpy")``):
|
|
6
|
+
365 or 360 day-of-year thresholds, years that end on the last day of December of the
|
|
7
|
+
calendar (``get.last.monthday.of.year``: 30 December for ``360_day``), and day numbers
|
|
8
|
+
``1..dpy`` (``as.POSIXlt(dates)$yday + 1``).
|
|
9
|
+
|
|
10
|
+
A :class:`CalendarDays` stands in for the :class:`pandas.DatetimeIndex` the index code
|
|
11
|
+
works with, since such dates (30 February) cannot be held by one. It has the parts of
|
|
12
|
+
its interface that code uses: ``year``, ``month``, ``day``, ``day_of_year``, the size,
|
|
13
|
+
indexing and slicing, and :meth:`CalendarDays.get_indexer`. Standard-calendar data
|
|
14
|
+
keep using a ``DatetimeIndex``, so nothing changes for them; the helpers here accept
|
|
15
|
+
either.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from collections.abc import Sequence
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
import pandas as pd
|
|
25
|
+
import xarray as xr
|
|
26
|
+
|
|
27
|
+
# CF calendar names, by the canonical name used here.
|
|
28
|
+
CALENDAR_NAMES = {
|
|
29
|
+
"noleap": {"noleap", "365_day"},
|
|
30
|
+
"360_day": {"360_day"},
|
|
31
|
+
}
|
|
32
|
+
STANDARD_CALENDARS = {"standard", "gregorian", "proleptic_gregorian"}
|
|
33
|
+
DAYS_PER_YEAR = {"noleap": 365, "360_day": 360}
|
|
34
|
+
_MONTH_LENGTHS = {
|
|
35
|
+
"noleap": np.array([31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]),
|
|
36
|
+
"360_day": np.full(12, 30),
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def canonical_calendar(name: str | None) -> str | None:
|
|
41
|
+
"""``"noleap"`` or ``"360_day"`` for those calendars (any CF spelling), None for the
|
|
42
|
+
standard calendar (or no calendar attribute); raises ``NotImplementedError`` for
|
|
43
|
+
any other (``all_leap``, ``julian``)."""
|
|
44
|
+
if name is None:
|
|
45
|
+
return None
|
|
46
|
+
key = str(name).strip().lower()
|
|
47
|
+
if key in STANDARD_CALENDARS:
|
|
48
|
+
return None
|
|
49
|
+
for canonical, names in CALENDAR_NAMES.items():
|
|
50
|
+
if key in names:
|
|
51
|
+
return canonical
|
|
52
|
+
raise NotImplementedError(f"the {name!r} calendar is not supported")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class CalendarDate:
|
|
57
|
+
"""One date of a :class:`CalendarDays`."""
|
|
58
|
+
|
|
59
|
+
year: int
|
|
60
|
+
month: int
|
|
61
|
+
day: int
|
|
62
|
+
|
|
63
|
+
def __str__(self) -> str:
|
|
64
|
+
return f"{self.year:04d}-{self.month:02d}-{self.day:02d}"
|
|
65
|
+
|
|
66
|
+
def __format__(self, spec: str) -> str:
|
|
67
|
+
"""``%Y``, ``%m`` and ``%d`` as ``strftime``; no spec gives ``YYYY-MM-DD``."""
|
|
68
|
+
if not spec:
|
|
69
|
+
return str(self)
|
|
70
|
+
return (
|
|
71
|
+
spec.replace("%Y", f"{self.year:04d}")
|
|
72
|
+
.replace("%m", f"{self.month:02d}")
|
|
73
|
+
.replace("%d", f"{self.day:02d}")
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class CalendarDays:
|
|
78
|
+
"""A daily time axis in the ``noleap`` or ``360_day`` calendar (see the module)."""
|
|
79
|
+
|
|
80
|
+
__slots__ = ("calendar", "_year", "_month", "_day")
|
|
81
|
+
|
|
82
|
+
def __init__(self, calendar: str, year, month, day):
|
|
83
|
+
self.calendar = calendar
|
|
84
|
+
self._year = np.asarray(year, dtype=np.int64)
|
|
85
|
+
self._month = np.asarray(month, dtype=np.int64)
|
|
86
|
+
self._day = np.asarray(day, dtype=np.int64)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def whole_years(cls, calendar: str, first: int, last: int) -> CalendarDays:
|
|
90
|
+
"""Every day from 1 January of ``first`` to the last day of ``last``."""
|
|
91
|
+
dpy = DAYS_PER_YEAR[calendar]
|
|
92
|
+
lengths = _MONTH_LENGTHS[calendar]
|
|
93
|
+
month = np.repeat(np.arange(1, 13), lengths)
|
|
94
|
+
day = np.concatenate([np.arange(1, n + 1) for n in lengths])
|
|
95
|
+
years = np.arange(first, last + 1)
|
|
96
|
+
return cls(
|
|
97
|
+
calendar,
|
|
98
|
+
np.repeat(years, dpy),
|
|
99
|
+
np.tile(month, years.size),
|
|
100
|
+
np.tile(day, years.size),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def from_cftime(cls, dates: Sequence, calendar: str) -> CalendarDays:
|
|
105
|
+
"""From cftime dates (truncated to days)."""
|
|
106
|
+
return cls(
|
|
107
|
+
calendar,
|
|
108
|
+
[d.year for d in dates],
|
|
109
|
+
[d.month for d in dates],
|
|
110
|
+
[d.day for d in dates],
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def dpy(self) -> int:
|
|
115
|
+
return DAYS_PER_YEAR[self.calendar]
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def year(self) -> pd.Index:
|
|
119
|
+
return pd.Index(self._year)
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def month(self) -> pd.Index:
|
|
123
|
+
return pd.Index(self._month)
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def day(self) -> pd.Index:
|
|
127
|
+
return pd.Index(self._day)
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def day_of_year(self) -> pd.Index:
|
|
131
|
+
"""1 to :attr:`dpy` (climdex.pcic ``get.jdays``)."""
|
|
132
|
+
start = np.concatenate([[0], np.cumsum(_MONTH_LENGTHS[self.calendar])[:-1]])
|
|
133
|
+
return pd.Index(start[self._month - 1] + self._day)
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def ordinal(self) -> np.ndarray:
|
|
137
|
+
"""Days since 1 January of year 0 in the calendar."""
|
|
138
|
+
return self._year * self.dpy + self.day_of_year.to_numpy() - 1
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def size(self) -> int:
|
|
142
|
+
return self._year.size
|
|
143
|
+
|
|
144
|
+
def __len__(self) -> int:
|
|
145
|
+
return self.size
|
|
146
|
+
|
|
147
|
+
def __getitem__(self, key):
|
|
148
|
+
if isinstance(key, (int, np.integer)):
|
|
149
|
+
return CalendarDate(int(self._year[key]), int(self._month[key]), int(self._day[key]))
|
|
150
|
+
return CalendarDays(self.calendar, self._year[key], self._month[key], self._day[key])
|
|
151
|
+
|
|
152
|
+
def __repr__(self) -> str:
|
|
153
|
+
if not self.size:
|
|
154
|
+
return f"CalendarDays({self.calendar!r}, empty)"
|
|
155
|
+
return f"CalendarDays({self.calendar!r}, {self[0]} .. {self[-1]}, {self.size} days)"
|
|
156
|
+
|
|
157
|
+
def get_indexer(self, target: CalendarDays) -> np.ndarray:
|
|
158
|
+
"""Position of each date of ``target`` in this axis, -1 where absent."""
|
|
159
|
+
return pd.Index(self.ordinal).get_indexer(target.ordinal)
|
|
160
|
+
|
|
161
|
+
def equals(self, other) -> bool:
|
|
162
|
+
return (
|
|
163
|
+
isinstance(other, CalendarDays)
|
|
164
|
+
and other.calendar == self.calendar
|
|
165
|
+
and np.array_equal(other.ordinal, self.ordinal)
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def is_unique(self) -> bool:
|
|
170
|
+
return pd.Index(self.ordinal).is_unique
|
|
171
|
+
|
|
172
|
+
def days_since(self, origin: CalendarDate) -> np.ndarray:
|
|
173
|
+
"""Days from ``origin`` (a date of the same calendar) to each date, as float."""
|
|
174
|
+
o = CalendarDays(self.calendar, [origin.year], [origin.month], [origin.day])
|
|
175
|
+
return (self.ordinal - o.ordinal[0]).astype(float)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
DayAxis = pd.DatetimeIndex | CalendarDays
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def as_days(time) -> DayAxis:
|
|
182
|
+
"""``time`` as a :class:`CalendarDays` (unchanged) or a ``DatetimeIndex``."""
|
|
183
|
+
if isinstance(time, CalendarDays):
|
|
184
|
+
return time
|
|
185
|
+
return pd.DatetimeIndex(time.values if isinstance(time, xr.DataArray) else time)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def calendar_of(time) -> str | None:
|
|
189
|
+
"""The calendar of a day axis: ``"noleap"``, ``"360_day"``, or None (standard)."""
|
|
190
|
+
return time.calendar if isinstance(time, CalendarDays) else None
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def days_per_year(calendar: str | None) -> int:
|
|
194
|
+
"""climdex.pcic's ``dpy``: 365 for the standard calendar (29 February dropped)."""
|
|
195
|
+
return 365 if calendar is None else DAYS_PER_YEAR[calendar]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def whole_years(time) -> DayAxis:
|
|
199
|
+
"""1 January of the first year of ``time`` to the last day of its last year
|
|
200
|
+
(climdex.pcic's ``date.series``), in its calendar."""
|
|
201
|
+
time = as_days(time)
|
|
202
|
+
if isinstance(time, CalendarDays):
|
|
203
|
+
return CalendarDays.whole_years(time.calendar, time[0].year, time[-1].year)
|
|
204
|
+
return pd.date_range(f"{time[0].year}-01-01", f"{time[-1].year}-12-31", freq="D")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def year_range(calendar: str | None, first: int, last: int) -> DayAxis:
|
|
208
|
+
"""Every day of the years ``first`` to ``last`` in ``calendar``."""
|
|
209
|
+
if calendar is None:
|
|
210
|
+
return pd.date_range(f"{first}-01-01", f"{last}-12-31", freq="D")
|
|
211
|
+
return CalendarDays.whole_years(calendar, first, last)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
__all__ = [
|
|
215
|
+
"CalendarDate",
|
|
216
|
+
"CalendarDays",
|
|
217
|
+
"DayAxis",
|
|
218
|
+
"as_days",
|
|
219
|
+
"calendar_of",
|
|
220
|
+
"canonical_calendar",
|
|
221
|
+
"days_per_year",
|
|
222
|
+
"whole_years",
|
|
223
|
+
"year_range",
|
|
224
|
+
]
|