cetk 0.0.4.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cetk/__init__.py +87 -0
- cetk/db.py +41 -0
- cetk/edb/__init__.py +0 -0
- cetk/edb/apps.py +9 -0
- cetk/edb/cache.py +249 -0
- cetk/edb/const.py +26 -0
- cetk/edb/copy.py +18 -0
- cetk/edb/exporters.py +551 -0
- cetk/edb/importers/__init__.py +36 -0
- cetk/edb/importers/activity_import.py +279 -0
- cetk/edb/importers/codeset_import.py +124 -0
- cetk/edb/importers/gridsource_import.py +256 -0
- cetk/edb/importers/roadsource_import.py +1602 -0
- cetk/edb/importers/source_import.py +958 -0
- cetk/edb/importers/timevar_import.py +115 -0
- cetk/edb/importers/utils.py +92 -0
- cetk/edb/importers/validation.py +207 -0
- cetk/edb/ltreefield.py +76 -0
- cetk/edb/migrations/0001_initial.py +1731 -0
- cetk/edb/migrations/0002_data.py +156 -0
- cetk/edb/migrations/0003_create_gpkg_tables.py +14 -0
- cetk/edb/migrations/__init__.py +0 -0
- cetk/edb/models/__init__.py +49 -0
- cetk/edb/models/base.py +52 -0
- cetk/edb/models/common_models.py +144 -0
- cetk/edb/models/fleets.py +160 -0
- cetk/edb/models/gridsource_models.py +163 -0
- cetk/edb/models/road_classes.py +197 -0
- cetk/edb/models/road_models.py +413 -0
- cetk/edb/models/source_models.py +399 -0
- cetk/edb/models/timevar_models.py +111 -0
- cetk/edb/rasterize/__init__.py +1 -0
- cetk/edb/rasterize/rasterizer.py +1544 -0
- cetk/edb/signals.py +43 -0
- cetk/edb/traffic.py +75 -0
- cetk/edb/units.py +160 -0
- cetk/emissions/__init__.py +0 -0
- cetk/emissions/aggregate_emissions.sql +303 -0
- cetk/emissions/areasource_emissions.sql +53 -0
- cetk/emissions/calc.py +148 -0
- cetk/emissions/filters.py +179 -0
- cetk/emissions/gridsource_emissions.sql +52 -0
- cetk/emissions/pointsource_emissions.sql +53 -0
- cetk/emissions/queries.py +221 -0
- cetk/emissions/roadsource_emissions.sql +211 -0
- cetk/emissions/used_substances.sql +56 -0
- cetk/emissions/views.py +89 -0
- cetk/logging.py +68 -0
- cetk/settings.py +85 -0
- cetk/tools/__init__.py +1 -0
- cetk/tools/cetk_command.py +599 -0
- cetk/tools/manage.py +24 -0
- cetk/tools/utils.py +334 -0
- cetk/utils.py +62 -0
- cetk-0.0.4.dev0.dist-info/LICENSE +201 -0
- cetk-0.0.4.dev0.dist-info/METADATA +297 -0
- cetk-0.0.4.dev0.dist-info/RECORD +60 -0
- cetk-0.0.4.dev0.dist-info/WHEEL +5 -0
- cetk-0.0.4.dev0.dist-info/entry_points.txt +3 -0
- cetk-0.0.4.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1544 @@
|
|
|
1
|
+
"""Rasterization of emission sources."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
from copy import copy
|
|
7
|
+
from functools import cache
|
|
8
|
+
from itertools import product
|
|
9
|
+
from math import ceil
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
import netCDF4 as nc
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import rasterio as rio
|
|
17
|
+
from django.contrib.gis.geos import GEOSGeometry, Polygon
|
|
18
|
+
from rastafari import ddaf_line_subpixel, even_odd_polygon_fill, resample_band
|
|
19
|
+
|
|
20
|
+
from cetk import __version__
|
|
21
|
+
from cetk.edb.cache import EmissionCache, NotInCacheError
|
|
22
|
+
from cetk.edb.models import (
|
|
23
|
+
VELOCITY_CHOICES,
|
|
24
|
+
AreaSource,
|
|
25
|
+
ColdstartTimevar,
|
|
26
|
+
CongestionProfile,
|
|
27
|
+
FlowTimevar,
|
|
28
|
+
GridSource,
|
|
29
|
+
Parameter,
|
|
30
|
+
PointSource,
|
|
31
|
+
RoadSource,
|
|
32
|
+
Settings,
|
|
33
|
+
Timevar,
|
|
34
|
+
get_gridsource_raster,
|
|
35
|
+
timevar_to_series,
|
|
36
|
+
)
|
|
37
|
+
from cetk.edb.models.gridsource_models import OutsideExtentError
|
|
38
|
+
from cetk.edb.traffic import los_to_velocity
|
|
39
|
+
from cetk.edb.units import emis_conversion_factor_from_si
|
|
40
|
+
from cetk.emissions.calc import calculate_source_emissions
|
|
41
|
+
|
|
42
|
+
# short source type identifiers for conveniency
|
|
43
|
+
POINT = PointSource.sourcetype
|
|
44
|
+
AREA = AreaSource.sourcetype
|
|
45
|
+
GRID = GridSource.sourcetype
|
|
46
|
+
ROAD = RoadSource.sourcetype
|
|
47
|
+
|
|
48
|
+
# supported source types
|
|
49
|
+
SOURCETYPES = (POINT, AREA, GRID, ROAD)
|
|
50
|
+
|
|
51
|
+
# slug for traffic work parameter and traffic condition parameter
|
|
52
|
+
TRAFFIC_WORK_PARAMETER_SLUG = "traffic_work"
|
|
53
|
+
TRAFFIC_CONDITION_PARAMETER_SLUG = "traffic_condition"
|
|
54
|
+
TRAFFIC_WORK_SUBSTANCE_SLUG = "traffic_work"
|
|
55
|
+
VEHICLE_SPEED_PARAMETER_SLUG = "vehicle_speed"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Output:
|
|
59
|
+
def __init__(
|
|
60
|
+
self, extent: tuple, path: Path, srid: int, timezone: datetime.timezone
|
|
61
|
+
):
|
|
62
|
+
self.extent = extent
|
|
63
|
+
self.path = path
|
|
64
|
+
self.srid = srid
|
|
65
|
+
self.timezone = timezone
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@cache
|
|
69
|
+
def time_step_as_delta(time_step):
|
|
70
|
+
"""Return timedelta corresponding to a time_step.
|
|
71
|
+
For Y and M, returns an approximate time-delta.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
if time_step.endswith("Y"):
|
|
75
|
+
nyears = int(time_step[:-1])
|
|
76
|
+
return pd.Timedelta(days=nyears * 365)
|
|
77
|
+
|
|
78
|
+
if time_step.endswith("M"):
|
|
79
|
+
nmonths = int(time_step[:-1])
|
|
80
|
+
return pd.Timedelta(days=nmonths * 30)
|
|
81
|
+
|
|
82
|
+
ts = time_step.lower() if time_step.endswith("h") else time_step
|
|
83
|
+
try:
|
|
84
|
+
return pd.Timedelta(ts)
|
|
85
|
+
except ValueError:
|
|
86
|
+
raise ValueError(f"invalid time step abbreviation {ts}")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class EmissionRasterizer:
|
|
90
|
+
"""An emission time-series rasterizer.
|
|
91
|
+
|
|
92
|
+
writes emissions to outputfile in units kg/s
|
|
93
|
+
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
def __init__(self, output, nx, ny):
|
|
97
|
+
"""
|
|
98
|
+
args
|
|
99
|
+
output: output.path: path to store NetCDF files
|
|
100
|
+
output.extent: extent of the raster (x1, y1, x2, y2)
|
|
101
|
+
output.srid: SRID of output, as int of 4 numbers (ex. 3006)
|
|
102
|
+
output.timezone: timezone of output
|
|
103
|
+
nx: raster dimension in x-direction
|
|
104
|
+
ny: raster dimension in y-direction
|
|
105
|
+
"""
|
|
106
|
+
self.timevars = {}
|
|
107
|
+
self.flow_timevars = {}
|
|
108
|
+
self.coldstart_timevars = {}
|
|
109
|
+
self.congestion_profiles = {}
|
|
110
|
+
self.level_weights = {}
|
|
111
|
+
self.output = output
|
|
112
|
+
self.extent = output.extent
|
|
113
|
+
self.crs = rio.crs.CRS.from_epsg(output.srid)
|
|
114
|
+
self._cache = None
|
|
115
|
+
self.nx = nx
|
|
116
|
+
self.ny = ny
|
|
117
|
+
self.sourcetypes = []
|
|
118
|
+
|
|
119
|
+
# querysets for emission
|
|
120
|
+
self.querysets = {}
|
|
121
|
+
|
|
122
|
+
self.log = logging.getLogger(__name__)
|
|
123
|
+
|
|
124
|
+
def reset(self):
|
|
125
|
+
"""Reset attributes.
|
|
126
|
+
To allow reuse of rasterizer (e.g. for a new substance)
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
self.querysets = {}
|
|
130
|
+
self.sourcetypes = []
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def x1(self):
|
|
134
|
+
return self.extent[0]
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def x2(self):
|
|
138
|
+
return self.extent[2]
|
|
139
|
+
|
|
140
|
+
@property
|
|
141
|
+
def y1(self):
|
|
142
|
+
return self.extent[1]
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def y2(self):
|
|
146
|
+
return self.extent[3]
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def dx(self):
|
|
150
|
+
return (self.x2 - self.x1) / self.nx
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def dy(self):
|
|
154
|
+
return (self.y2 - self.y1) / self.ny
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def srid(self):
|
|
158
|
+
return self.output.srid
|
|
159
|
+
|
|
160
|
+
def _get_timevars(self, sourcetypes):
|
|
161
|
+
"""Get all time-variations."""
|
|
162
|
+
self.timevars = {}
|
|
163
|
+
self.flow_timevars = {}
|
|
164
|
+
self.coldstart_timevars = {}
|
|
165
|
+
self.congestion_profiles = {}
|
|
166
|
+
|
|
167
|
+
if POINT in sourcetypes or AREA in sourcetypes:
|
|
168
|
+
for tvar in Timevar.objects.all():
|
|
169
|
+
self.timevars[tvar.id] = tvar
|
|
170
|
+
default = Timevar()
|
|
171
|
+
self.timevars["default"] = default
|
|
172
|
+
|
|
173
|
+
if ROAD in sourcetypes:
|
|
174
|
+
for tvar in FlowTimevar.objects.all():
|
|
175
|
+
self.flow_timevars[tvar.id] = tvar
|
|
176
|
+
default = FlowTimevar()
|
|
177
|
+
self.flow_timevars["default"] = default
|
|
178
|
+
|
|
179
|
+
for tvar in ColdstartTimevar.objects.all():
|
|
180
|
+
self.coldstart_timevars[tvar.id] = tvar
|
|
181
|
+
default = ColdstartTimevar()
|
|
182
|
+
self.coldstart_timevars["default"] = default
|
|
183
|
+
|
|
184
|
+
for prof in CongestionProfile.objects.all():
|
|
185
|
+
self.congestion_profiles[prof.id] = prof
|
|
186
|
+
default = CongestionProfile()
|
|
187
|
+
self.congestion_profiles["default"] = default
|
|
188
|
+
|
|
189
|
+
def _get_level_weights(self):
|
|
190
|
+
"""redistribute vertical distr. into levels of result dataset."""
|
|
191
|
+
# TODO don't use levels yet
|
|
192
|
+
if Settings.get_current().codeset1 is None:
|
|
193
|
+
self.log.warning(
|
|
194
|
+
"inventory has no primary code-set - "
|
|
195
|
+
"cannot apply vertical distributions "
|
|
196
|
+
)
|
|
197
|
+
return
|
|
198
|
+
|
|
199
|
+
bounds = self.levels[:]
|
|
200
|
+
|
|
201
|
+
weights = np.zeros(self.levels.shape, dtype=float)
|
|
202
|
+
# cache vertical distributions to used for different activitycodes
|
|
203
|
+
# heights are determined using the 1st (primary) codeset
|
|
204
|
+
for ac in (
|
|
205
|
+
Settings.get_current()
|
|
206
|
+
.codeset1.codes.prefetch_related("vertical_dist")
|
|
207
|
+
.exclude(vertical_dist__isnull=True)
|
|
208
|
+
):
|
|
209
|
+
# eval is needed in cetk because arrays stored as CharField in sqlite
|
|
210
|
+
vdist = np.array(eval(ac.vertical_dist.weights))
|
|
211
|
+
vdist_weights = vdist[:, 1]
|
|
212
|
+
vdist_heights = vdist[:, 0]
|
|
213
|
+
|
|
214
|
+
if vdist_heights[0] > 0:
|
|
215
|
+
vdist_weights = np.insert(vdist_weights, 0, 0)
|
|
216
|
+
vdist_heights = np.insert(vdist_heights, 0, 0)
|
|
217
|
+
|
|
218
|
+
# interp weights at level bounds
|
|
219
|
+
interp_weights = np.interp(bounds, vdist_heights, vdist_weights, right=0)
|
|
220
|
+
# insert interpolated values into vdist
|
|
221
|
+
start_ind = 0
|
|
222
|
+
level = 0
|
|
223
|
+
for bound, val in zip(bounds, interp_weights):
|
|
224
|
+
# index where bound will be inserted
|
|
225
|
+
ind = np.searchsorted(vdist_heights, bound)
|
|
226
|
+
if bound not in vdist_heights:
|
|
227
|
+
vdist_weights = np.insert(vdist_weights, ind, val)
|
|
228
|
+
vdist_heights = np.insert(vdist_heights, ind, bound)
|
|
229
|
+
# integrate weight between level bounds
|
|
230
|
+
weight = np.trapz(
|
|
231
|
+
vdist_weights[start_ind : ind + 1],
|
|
232
|
+
vdist_heights[start_ind : ind + 1],
|
|
233
|
+
)
|
|
234
|
+
start_ind = ind
|
|
235
|
+
weights[level] = weight
|
|
236
|
+
level += 1
|
|
237
|
+
weights_sum = weights.sum()
|
|
238
|
+
if weights_sum != 0:
|
|
239
|
+
weights /= weights_sum
|
|
240
|
+
vertical_index = weights.nonzero()
|
|
241
|
+
non_zero_weights = weights[vertical_index]
|
|
242
|
+
self.level_weights[ac.code] = (vertical_index[0], non_zero_weights[:])
|
|
243
|
+
|
|
244
|
+
def _timevariations_to_dataframe(self, begin, end):
|
|
245
|
+
"""Prepare hourly dataframes in UTC for all time-variation profiles.
|
|
246
|
+
|
|
247
|
+
args
|
|
248
|
+
begin: start time in UTC
|
|
249
|
+
end: end time in UTC
|
|
250
|
+
|
|
251
|
+
"""
|
|
252
|
+
time_index = pd.date_range(begin, end, freq="h")
|
|
253
|
+
one_hour = datetime.timedelta(hours=1)
|
|
254
|
+
shifted_index = pd.date_range(begin - one_hour, end - one_hour, freq="h")
|
|
255
|
+
|
|
256
|
+
self.flow_timevar_scalings = pd.DataFrame(
|
|
257
|
+
{
|
|
258
|
+
fvar.id
|
|
259
|
+
or "default": timevar_to_series(
|
|
260
|
+
shifted_index, fvar, timezone=self.timezone
|
|
261
|
+
).to_numpy() # discard the shifted index by converting to ndarray
|
|
262
|
+
for fvar in self.flow_timevars.values()
|
|
263
|
+
},
|
|
264
|
+
index=time_index,
|
|
265
|
+
)
|
|
266
|
+
self.traffic_conditions = pd.DataFrame(
|
|
267
|
+
{
|
|
268
|
+
prof.id
|
|
269
|
+
or "default": prof.to_series(
|
|
270
|
+
shifted_index, timezone=self.timezone
|
|
271
|
+
).to_numpy()
|
|
272
|
+
for prof in self.congestion_profiles.values()
|
|
273
|
+
},
|
|
274
|
+
index=time_index,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
# calculate vehicle velocity from level-of-service and posted speed
|
|
278
|
+
self.velocity_light = {}
|
|
279
|
+
self.velocity_heavy = {}
|
|
280
|
+
for key, vel in product(self.traffic_conditions, VELOCITY_CHOICES):
|
|
281
|
+
lvehspd, hvehspd = los_to_velocity(self.traffic_conditions[key], vel)
|
|
282
|
+
# convert to SI units, km/h to m/s
|
|
283
|
+
self.velocity_light[(key, vel)] = lvehspd / 3.6
|
|
284
|
+
self.velocity_heavy[(key, vel)] = hvehspd / 3.6
|
|
285
|
+
|
|
286
|
+
self.source_timevar_scalings = pd.DataFrame(
|
|
287
|
+
{
|
|
288
|
+
tvar.id
|
|
289
|
+
or "default": timevar_to_series(
|
|
290
|
+
shifted_index, tvar, timezone=self.output.timezone
|
|
291
|
+
).to_numpy()
|
|
292
|
+
for tvar in self.timevars.values()
|
|
293
|
+
},
|
|
294
|
+
index=time_index,
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
def _get_querysets( # noqa: C901, PLR0912
|
|
298
|
+
self,
|
|
299
|
+
sourcetypes=None,
|
|
300
|
+
name=None,
|
|
301
|
+
tags=None,
|
|
302
|
+
point_ids=None,
|
|
303
|
+
area_ids=None,
|
|
304
|
+
grid_ids=None,
|
|
305
|
+
road_ids=None,
|
|
306
|
+
polygon=None,
|
|
307
|
+
substances=None,
|
|
308
|
+
ac1=None,
|
|
309
|
+
ac2=None,
|
|
310
|
+
ac3=None,
|
|
311
|
+
*,
|
|
312
|
+
cur=None,
|
|
313
|
+
):
|
|
314
|
+
"""Get querysets for emissions."""
|
|
315
|
+
|
|
316
|
+
sourcetypes = sourcetypes or SOURCETYPES
|
|
317
|
+
# not used; srid = Settings.get_current().srid
|
|
318
|
+
|
|
319
|
+
if ac1 is not None or ac2 is not None or ac3 is not None:
|
|
320
|
+
raise ValueError(
|
|
321
|
+
"filtering for ac not implemented in source_emissions_query yet!"
|
|
322
|
+
)
|
|
323
|
+
# if activity codes are given as strings, get the corresponding
|
|
324
|
+
# activity code instances
|
|
325
|
+
if ac1 is not None and len(ac1) > 0 and isinstance(ac1[0], str):
|
|
326
|
+
# TODO why code__in= not code= ?
|
|
327
|
+
ac1_instances = list(
|
|
328
|
+
Settings.get_current().codeset1.codes.filter(code__in=ac1)
|
|
329
|
+
)
|
|
330
|
+
if len(ac1_instances) == 0:
|
|
331
|
+
raise ValueError(
|
|
332
|
+
f"the filter for activitycode1: {ac1} does not match any code"
|
|
333
|
+
f" in code-set '{Settings.get_current().codeset1.name}'"
|
|
334
|
+
)
|
|
335
|
+
else:
|
|
336
|
+
ac1_instances = ac1
|
|
337
|
+
|
|
338
|
+
if ac2 is not None and len(ac2) > 0 and isinstance(ac2[0], str):
|
|
339
|
+
ac2_instances = list(
|
|
340
|
+
Settings.get_current().codeset2.codes.filter(code__in=ac2)
|
|
341
|
+
)
|
|
342
|
+
if len(ac2_instances) == 0:
|
|
343
|
+
raise ValueError(
|
|
344
|
+
f"the filter for activitycode2: {ac2} does not match any code"
|
|
345
|
+
f" in code-set '{Settings.get_current().codeset2.name}'"
|
|
346
|
+
)
|
|
347
|
+
else:
|
|
348
|
+
ac2_instances = ac2
|
|
349
|
+
|
|
350
|
+
if ac3 is not None and len(ac3) > 0 and isinstance(ac3[0], str):
|
|
351
|
+
ac3_instances = list(
|
|
352
|
+
Settings.get_current().codeset3.codes.filter(code__in=ac3)
|
|
353
|
+
)
|
|
354
|
+
if len(ac3_instances) == 0:
|
|
355
|
+
raise ValueError(
|
|
356
|
+
f"the filter for activitycode3: {ac3} does not match any code"
|
|
357
|
+
f" in code-set '{Settings.get_current().codeset3.name}'"
|
|
358
|
+
)
|
|
359
|
+
else:
|
|
360
|
+
ac3_instances = ac3
|
|
361
|
+
|
|
362
|
+
# aggregate emissions for each source-type per sourceid
|
|
363
|
+
# TODO, ac1-3 and tolerance not used in calculate_source_emissions yet
|
|
364
|
+
# tolerance = None
|
|
365
|
+
for sourcetype in sourcetypes:
|
|
366
|
+
self.log.debug(f"- {sourcetype}source emissions")
|
|
367
|
+
if sourcetype == POINT:
|
|
368
|
+
if not PointSource.objects.exists():
|
|
369
|
+
continue
|
|
370
|
+
ids = point_ids
|
|
371
|
+
elif sourcetype == AREA:
|
|
372
|
+
if not AreaSource.objects.exists():
|
|
373
|
+
continue
|
|
374
|
+
ids = area_ids
|
|
375
|
+
elif sourcetype == ROAD:
|
|
376
|
+
if not RoadSource.objects.exists():
|
|
377
|
+
continue
|
|
378
|
+
ids = road_ids
|
|
379
|
+
elif sourcetype == GRID:
|
|
380
|
+
if not GridSource.objects.exists():
|
|
381
|
+
continue
|
|
382
|
+
ids = grid_ids
|
|
383
|
+
else:
|
|
384
|
+
raise ValueError(
|
|
385
|
+
f"Rasterize cannot handle sourcetype {sourcetype} yet."
|
|
386
|
+
)
|
|
387
|
+
self.querysets[sourcetype] = calculate_source_emissions(
|
|
388
|
+
sourcetype,
|
|
389
|
+
substances=substances,
|
|
390
|
+
srid=int(self.output.srid),
|
|
391
|
+
name=name,
|
|
392
|
+
ids=ids,
|
|
393
|
+
tags=tags,
|
|
394
|
+
polygon=polygon,
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
def _get_weights(self, polygon=None):
|
|
398
|
+
"""Get cell weights for all sourcetypes."""
|
|
399
|
+
|
|
400
|
+
for sourcetype in self.querysets:
|
|
401
|
+
if sourcetype == POINT:
|
|
402
|
+
self._get_point_weights()
|
|
403
|
+
elif sourcetype == AREA:
|
|
404
|
+
self._get_area_weights()
|
|
405
|
+
if sourcetype == ROAD:
|
|
406
|
+
self._get_road_weights()
|
|
407
|
+
elif sourcetype == GRID:
|
|
408
|
+
self._get_grid_weights(polygon=polygon)
|
|
409
|
+
|
|
410
|
+
def _get_point_weights(self):
|
|
411
|
+
"""Get cell weights for points.
|
|
412
|
+
returns a dict with source_id as keys and
|
|
413
|
+
(index_array, 1.0) as values
|
|
414
|
+
|
|
415
|
+
complete records from query result are stored in dict
|
|
416
|
+
with source_id as key.
|
|
417
|
+
"""
|
|
418
|
+
col_map = self._cache.col_maps[POINT]
|
|
419
|
+
for rec in self.querysets[POINT]:
|
|
420
|
+
source_id = rec[col_map.source_id]
|
|
421
|
+
|
|
422
|
+
if source_id not in self._cache.gridded_sources[POINT]:
|
|
423
|
+
[[x, y]] = np.array(get_nodes_from_wkt(rec[col_map.wkt]))
|
|
424
|
+
|
|
425
|
+
# check if source is within bounds
|
|
426
|
+
if (x < self.x1 or y < self.y1) or (x > self.x2 or y > self.y2):
|
|
427
|
+
continue
|
|
428
|
+
|
|
429
|
+
col = min(int((x - self.x1) / self.dx), self.nx - 1)
|
|
430
|
+
row = min(self.ny - int(ceil((y - self.y1) / self.dy)), self.ny - 1)
|
|
431
|
+
index = ((row,), (col,))
|
|
432
|
+
|
|
433
|
+
# stores index array for consistency with other sourcetypes
|
|
434
|
+
weights = (index, 1.0)
|
|
435
|
+
self._cache.add_rec(rec, POINT, write_weights=True, weights=weights)
|
|
436
|
+
else:
|
|
437
|
+
self._cache.add_rec(rec, POINT, write_weights=False)
|
|
438
|
+
|
|
439
|
+
# write last page to cache
|
|
440
|
+
self._cache.write_weights(POINT)
|
|
441
|
+
self._cache.write_emissions(POINT)
|
|
442
|
+
|
|
443
|
+
def _get_area_weights(self):
|
|
444
|
+
"""Get cell weights for area sources.
|
|
445
|
+
returns a dict with source_id as keys and
|
|
446
|
+
(index_array, weight_array) as values
|
|
447
|
+
|
|
448
|
+
complete records from query result are stored in a dict
|
|
449
|
+
with source_id as key.
|
|
450
|
+
"""
|
|
451
|
+
|
|
452
|
+
col_map = self._cache.col_maps[AREA]
|
|
453
|
+
for rec in self.querysets[AREA]:
|
|
454
|
+
source_id = rec[col_map.source_id]
|
|
455
|
+
|
|
456
|
+
if source_id not in self._cache.gridded_sources[AREA]:
|
|
457
|
+
source_weights = {}
|
|
458
|
+
# extract nodes from area geometry in WKT format
|
|
459
|
+
wkt = rec[col_map.wkt]
|
|
460
|
+
nodes = np.array(get_nodes_from_wkt(wkt))
|
|
461
|
+
|
|
462
|
+
is_within_extent = any(
|
|
463
|
+
self.x1 <= x <= self.x2 and self.y1 <= y <= self.y2
|
|
464
|
+
for x, y in nodes
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
if is_within_extent:
|
|
468
|
+
even_odd_polygon_fill(
|
|
469
|
+
nodes,
|
|
470
|
+
source_weights,
|
|
471
|
+
self.extent,
|
|
472
|
+
self.nx,
|
|
473
|
+
self.ny,
|
|
474
|
+
subgridcells=2,
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
if len(source_weights) > 0:
|
|
478
|
+
indices = list(zip(*source_weights.keys()))
|
|
479
|
+
|
|
480
|
+
index = (indices[0], indices[1])
|
|
481
|
+
|
|
482
|
+
weights = (
|
|
483
|
+
index,
|
|
484
|
+
np.fromiter(source_weights.values(), dtype=float),
|
|
485
|
+
)
|
|
486
|
+
else:
|
|
487
|
+
weights = None
|
|
488
|
+
self._cache.add_rec(rec, AREA, write_weights=True, weights=weights)
|
|
489
|
+
else:
|
|
490
|
+
self._cache.add_rec(rec, AREA, write_weights=False)
|
|
491
|
+
|
|
492
|
+
# write last cache page to disk
|
|
493
|
+
self._cache.write_weights(AREA)
|
|
494
|
+
self._cache.write_emissions(AREA)
|
|
495
|
+
|
|
496
|
+
def _get_road_weights(self):
|
|
497
|
+
"""Get cell weights for roads.
|
|
498
|
+
returns a dict with road_id as keys and
|
|
499
|
+
(index_array, weight_array) as values
|
|
500
|
+
|
|
501
|
+
complete records from query result are stored in dict
|
|
502
|
+
with road_id as key.
|
|
503
|
+
"""
|
|
504
|
+
|
|
505
|
+
col_map = self._cache.col_maps[ROAD]
|
|
506
|
+
for rec in self.querysets[ROAD]:
|
|
507
|
+
source_id = rec[col_map.source_id]
|
|
508
|
+
if source_id not in self._cache.gridded_sources[ROAD]:
|
|
509
|
+
road_weights = {}
|
|
510
|
+
# extract nodes from road geometry in WKT format
|
|
511
|
+
wkt = rec[col_map.wkt]
|
|
512
|
+
nodes = get_nodes_from_wkt(wkt)
|
|
513
|
+
|
|
514
|
+
# if geometry is simplified, the length of the simplified
|
|
515
|
+
# geometry should be used when rasterizing
|
|
516
|
+
try:
|
|
517
|
+
road_length = rec[col_map.simple_length]
|
|
518
|
+
except AttributeError:
|
|
519
|
+
road_length = rec[col_map.length]
|
|
520
|
+
|
|
521
|
+
# sum weights for each segment to each cell index
|
|
522
|
+
# (many segments can intersect the same cell)
|
|
523
|
+
|
|
524
|
+
for i in range(1, len(nodes)):
|
|
525
|
+
p1_x, p1_y = nodes[i - 1]
|
|
526
|
+
p2_x, p2_y = nodes[i]
|
|
527
|
+
|
|
528
|
+
# rasterize segment and add weights to road_weights dict
|
|
529
|
+
# for each segment the weights are multiplied with
|
|
530
|
+
# the segment fraction of the road
|
|
531
|
+
# the resulting road weights will sum up to 1
|
|
532
|
+
ddaf_line_subpixel(
|
|
533
|
+
p1_x,
|
|
534
|
+
p1_y,
|
|
535
|
+
p2_x,
|
|
536
|
+
p2_y,
|
|
537
|
+
road_weights,
|
|
538
|
+
road_length,
|
|
539
|
+
self.extent,
|
|
540
|
+
self.dx,
|
|
541
|
+
self.dy,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
if len(road_weights) > 0:
|
|
545
|
+
indices = list(zip(*road_weights.keys()))
|
|
546
|
+
|
|
547
|
+
try:
|
|
548
|
+
ncells = indices[0].size
|
|
549
|
+
except AttributeError:
|
|
550
|
+
ncells = len(indices[0])
|
|
551
|
+
if self.levels is not None:
|
|
552
|
+
# road sources are added to lowest level
|
|
553
|
+
level = 0
|
|
554
|
+
# add z dimension to index array
|
|
555
|
+
index = (tuple([level] * ncells), indices[0], indices[1])
|
|
556
|
+
else:
|
|
557
|
+
index = (indices[0], indices[1])
|
|
558
|
+
|
|
559
|
+
# store cell indices and a weight array for each road id
|
|
560
|
+
weights = (
|
|
561
|
+
index,
|
|
562
|
+
np.fromiter(road_weights.values(), dtype=float),
|
|
563
|
+
)
|
|
564
|
+
else:
|
|
565
|
+
weights = None
|
|
566
|
+
self._cache.add_rec(rec, ROAD, write_weights=True, weights=weights)
|
|
567
|
+
else:
|
|
568
|
+
self._cache.add_rec(rec, ROAD, write_weights=False)
|
|
569
|
+
|
|
570
|
+
# write last cache page to disk
|
|
571
|
+
self._cache.write_weights(ROAD)
|
|
572
|
+
# write a separate file for records of each substance
|
|
573
|
+
self._cache.write_emissions(ROAD)
|
|
574
|
+
|
|
575
|
+
def _get_grid_weights(self, polygon=None):
|
|
576
|
+
"""Store records of grid-sources in dict with source_id as key.
|
|
577
|
+
|
|
578
|
+
returns a dict with source_id as keys and
|
|
579
|
+
(index_array, weight_array) as values.
|
|
580
|
+
|
|
581
|
+
Complete records from query
|
|
582
|
+
result are stored in a dict with source_id as key.
|
|
583
|
+
"""
|
|
584
|
+
col_map = self._cache.col_maps[GRID]
|
|
585
|
+
for rec in self.querysets[GRID]:
|
|
586
|
+
source_id = rec[col_map.source_id]
|
|
587
|
+
raster_name = rec[col_map.raster]
|
|
588
|
+
# spatial distribution of a grid is unique for each raster
|
|
589
|
+
source_key = (source_id, raster_name)
|
|
590
|
+
if source_key not in self._cache.gridded_sources[GRID]:
|
|
591
|
+
try:
|
|
592
|
+
raster_data, metadata = get_gridsource_raster(
|
|
593
|
+
raster_name, clip_by=polygon
|
|
594
|
+
)
|
|
595
|
+
except OutsideExtentError:
|
|
596
|
+
# no overlap between query and this grid,
|
|
597
|
+
# try next gridsource
|
|
598
|
+
break
|
|
599
|
+
|
|
600
|
+
index_array, source_weights = resample_band(
|
|
601
|
+
raster_data,
|
|
602
|
+
metadata["extent"],
|
|
603
|
+
metadata["nodata"],
|
|
604
|
+
self.extent,
|
|
605
|
+
self.nx,
|
|
606
|
+
self.ny,
|
|
607
|
+
metadata["srid"],
|
|
608
|
+
self.srid,
|
|
609
|
+
subgridcells=2,
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
if len(source_weights) > 0:
|
|
613
|
+
if self.levels is not None:
|
|
614
|
+
# add weights in vertical direction
|
|
615
|
+
ac1 = rec[col_map.ac1]
|
|
616
|
+
|
|
617
|
+
if ac1 is not None:
|
|
618
|
+
# add indices and weights for vertical levels
|
|
619
|
+
try:
|
|
620
|
+
levels, level_weights = self.level_weights[ac1]
|
|
621
|
+
except KeyError:
|
|
622
|
+
# if no vertical distribution is assigned to the
|
|
623
|
+
# activity code, all emissions are put in the
|
|
624
|
+
# lowest layer
|
|
625
|
+
levels = np.array([0])
|
|
626
|
+
level_weights = np.ones(1, dtype=float)
|
|
627
|
+
index_array = (
|
|
628
|
+
np.tile(levels, index_array[0].size),
|
|
629
|
+
np.repeat(index_array[0], levels.size),
|
|
630
|
+
np.repeat(index_array[1], levels.size),
|
|
631
|
+
)
|
|
632
|
+
# weight each level with corresponding level-weight
|
|
633
|
+
source_weights = source_weights.repeat(
|
|
634
|
+
level_weights.size
|
|
635
|
+
) * np.tile(level_weights, source_weights.size)
|
|
636
|
+
else:
|
|
637
|
+
index_array = (
|
|
638
|
+
np.zeros(index_array.size[0]),
|
|
639
|
+
index_array[0],
|
|
640
|
+
index_array[1],
|
|
641
|
+
)
|
|
642
|
+
weights = (index_array, source_weights)
|
|
643
|
+
else:
|
|
644
|
+
weights = None
|
|
645
|
+
|
|
646
|
+
# store emission record
|
|
647
|
+
self._cache.add_rec(
|
|
648
|
+
rec,
|
|
649
|
+
GRID,
|
|
650
|
+
write_weights=True,
|
|
651
|
+
weights=weights,
|
|
652
|
+
)
|
|
653
|
+
else:
|
|
654
|
+
self._cache.add_rec(rec, GRID)
|
|
655
|
+
|
|
656
|
+
# write last cache page to disk
|
|
657
|
+
self._cache.write_weights(GRID)
|
|
658
|
+
self._cache.write_emissions(GRID)
|
|
659
|
+
|
|
660
|
+
def _create_variables( # noqa: C901, PLR0912, PLR0915
|
|
661
|
+
self, substances, *, timeseries=True
|
|
662
|
+
):
|
|
663
|
+
"""create netCDF variables."""
|
|
664
|
+
if timeseries:
|
|
665
|
+
cell_methods = "1h-mean"
|
|
666
|
+
time = True
|
|
667
|
+
self.time_step = "1h"
|
|
668
|
+
else:
|
|
669
|
+
cell_methods = None
|
|
670
|
+
time = False
|
|
671
|
+
self.time_step = None
|
|
672
|
+
|
|
673
|
+
self.variables = {}
|
|
674
|
+
for substance in substances:
|
|
675
|
+
result_file = os.path.join(self.output.path, substance.slug + ".nc")
|
|
676
|
+
with nc.Dataset(result_file, "w", format="NETCDF4") as dset:
|
|
677
|
+
write_general_attrs(dset)
|
|
678
|
+
grid_mapping_var = self.create_gridmapping_variable(dset, self.crs)
|
|
679
|
+
time_var, time_bounds_var = create_time_variable(dset)
|
|
680
|
+
create_xy_variables(dset, self.extent, self.crs, self.nx, self.ny)
|
|
681
|
+
subst_vars = self.variables.setdefault(substance.slug, {})
|
|
682
|
+
# create variables also without emission in extent
|
|
683
|
+
# if any(
|
|
684
|
+
# self._cache.has_substance(sourcetype, substance.id)
|
|
685
|
+
# for sourcetype in self.sourcetypes
|
|
686
|
+
# ):
|
|
687
|
+
param = Parameter.objects.get(quantity="emission", substance=substance)
|
|
688
|
+
chunking = self._calc_chunking(
|
|
689
|
+
chunk_cache=1e8,
|
|
690
|
+
)
|
|
691
|
+
var_name = param.slug
|
|
692
|
+
create_variable(
|
|
693
|
+
dset,
|
|
694
|
+
grid_mapping_var,
|
|
695
|
+
name=var_name,
|
|
696
|
+
unit=self.unit,
|
|
697
|
+
instance=self.instance,
|
|
698
|
+
cell_methods=cell_methods,
|
|
699
|
+
parameter=param.name,
|
|
700
|
+
time=time,
|
|
701
|
+
chunksizes=chunking,
|
|
702
|
+
)
|
|
703
|
+
subst_vars["field2d"] = {"emission": var_name}
|
|
704
|
+
ncreated = 0
|
|
705
|
+
for subst_vars in self.variables.values():
|
|
706
|
+
for sourcetype_vars in subst_vars.values():
|
|
707
|
+
ncreated += len(sourcetype_vars)
|
|
708
|
+
return ncreated > 0
|
|
709
|
+
|
|
710
|
+
def process( # noqa: PLR0915
|
|
711
|
+
self,
|
|
712
|
+
substances,
|
|
713
|
+
begin=None,
|
|
714
|
+
end=None,
|
|
715
|
+
sourcetypes=None,
|
|
716
|
+
exclude_points=False,
|
|
717
|
+
exclude_roads=False,
|
|
718
|
+
point_ids=None,
|
|
719
|
+
area_ids=None,
|
|
720
|
+
grid_ids=None,
|
|
721
|
+
road_ids=None,
|
|
722
|
+
name=None,
|
|
723
|
+
tags=None,
|
|
724
|
+
polygon=None,
|
|
725
|
+
ac1=None,
|
|
726
|
+
ac2=None,
|
|
727
|
+
ac3=None,
|
|
728
|
+
cur=None,
|
|
729
|
+
unit=None,
|
|
730
|
+
aggregation=None,
|
|
731
|
+
instance=None,
|
|
732
|
+
):
|
|
733
|
+
"""Write emissions to dataset.
|
|
734
|
+
|
|
735
|
+
args:
|
|
736
|
+
substances: an iterable of (or single) Substance model instances
|
|
737
|
+
|
|
738
|
+
optional args:
|
|
739
|
+
begin: datetime object in utc for first time rasterize
|
|
740
|
+
end: datetime object in utc for last time to rasterize
|
|
741
|
+
sourcetypes: 'point', 'area', 'grid' or 'road'
|
|
742
|
+
exclude_points: point source emissions not rasterized
|
|
743
|
+
exclude_roads: road source emissions not rasterized
|
|
744
|
+
road_ids: list of road id's to include
|
|
745
|
+
point_ids: list of road id's to include
|
|
746
|
+
area_ids: list of road id's to include
|
|
747
|
+
grid_ids: list of road id's to include
|
|
748
|
+
name: source name (accepts regexp)
|
|
749
|
+
tags: dictionary with tags and values to filter on
|
|
750
|
+
polygon: only include sources within the provided polygon
|
|
751
|
+
ac1: iterable of activity codes
|
|
752
|
+
ac2: iterable of activity codes
|
|
753
|
+
ac3: iterable of activity codes
|
|
754
|
+
cur: db cursor object, used to execute query if provided
|
|
755
|
+
unit: emission output unit
|
|
756
|
+
aggregation: aggregation label of result variables
|
|
757
|
+
instance: write emissions to specified variable instance
|
|
758
|
+
"""
|
|
759
|
+
self.instance = instance
|
|
760
|
+
self.aggregation = aggregation
|
|
761
|
+
self.levels = None
|
|
762
|
+
if polygon is None:
|
|
763
|
+
# output extent
|
|
764
|
+
x1, y1, x2, y2 = self.extent
|
|
765
|
+
polygon = Polygon(
|
|
766
|
+
((x1, y1), (x2, y1), (x2, y2), (x1, y2), (x1, y1)),
|
|
767
|
+
srid=int(self.output.srid),
|
|
768
|
+
)
|
|
769
|
+
self.unit = unit or "kg/s"
|
|
770
|
+
self.unit_conversion_factor = emis_conversion_factor_from_si(self.unit)
|
|
771
|
+
|
|
772
|
+
# determine which sourcetypes should be rasterized
|
|
773
|
+
self.sourcetypes = copy(sourcetypes) or list(SOURCETYPES)
|
|
774
|
+
if exclude_points and POINT in self.sourcetypes:
|
|
775
|
+
self.sourcetypes.remove(POINT)
|
|
776
|
+
if exclude_roads and ROAD in self.sourcetypes:
|
|
777
|
+
self.sourcetypes.remove(ROAD)
|
|
778
|
+
|
|
779
|
+
if not hasattr(substances, "__iter__"):
|
|
780
|
+
self.substances = [substances]
|
|
781
|
+
else:
|
|
782
|
+
self.substances = substances
|
|
783
|
+
|
|
784
|
+
self.log.debug("reading records from database")
|
|
785
|
+
|
|
786
|
+
# create querysets for all sources
|
|
787
|
+
self._get_querysets(
|
|
788
|
+
sourcetypes=sourcetypes,
|
|
789
|
+
name=name,
|
|
790
|
+
tags=tags,
|
|
791
|
+
point_ids=point_ids,
|
|
792
|
+
area_ids=area_ids,
|
|
793
|
+
road_ids=road_ids,
|
|
794
|
+
grid_ids=grid_ids,
|
|
795
|
+
polygon=polygon,
|
|
796
|
+
ac1=ac1,
|
|
797
|
+
ac2=ac2,
|
|
798
|
+
ac3=ac3,
|
|
799
|
+
cur=cur,
|
|
800
|
+
)
|
|
801
|
+
try:
|
|
802
|
+
Path(self.output.path).mkdir(exist_ok=True)
|
|
803
|
+
except FileNotFoundError:
|
|
804
|
+
raise FileNotFoundError(
|
|
805
|
+
"parent directory of rasterized output does not exist "
|
|
806
|
+
f"({str(self.output.path)})"
|
|
807
|
+
)
|
|
808
|
+
|
|
809
|
+
with EmissionCache(self.querysets) as cache:
|
|
810
|
+
self._cache = cache
|
|
811
|
+
|
|
812
|
+
self.log.debug("processing static data")
|
|
813
|
+
# calculate raster cell weights for sources in querysets
|
|
814
|
+
# and store in a dict
|
|
815
|
+
self._get_weights(polygon=polygon)
|
|
816
|
+
|
|
817
|
+
self.timezone = self.output.timezone
|
|
818
|
+
|
|
819
|
+
if begin is not None and end is not None:
|
|
820
|
+
self.log.debug("creating result variables")
|
|
821
|
+
created = self._create_variables(self.substances, timeseries=True)
|
|
822
|
+
|
|
823
|
+
# get time-variation profiles
|
|
824
|
+
self._get_timevars(sourcetypes or SOURCETYPES)
|
|
825
|
+
self.log.debug("calculating time-series emissions")
|
|
826
|
+
self._process_timeseries(begin, end)
|
|
827
|
+
else:
|
|
828
|
+
self.log.debug("creating result variables")
|
|
829
|
+
created = self._create_variables(self.substances, timeseries=False)
|
|
830
|
+
# if no variables are created, skip out
|
|
831
|
+
if not created:
|
|
832
|
+
self.reset()
|
|
833
|
+
self.log.info("no emissions found within requested extent")
|
|
834
|
+
return
|
|
835
|
+
self.log.debug("calculating average emissions")
|
|
836
|
+
self._process_average_emissions()
|
|
837
|
+
|
|
838
|
+
self.reset()
|
|
839
|
+
|
|
840
|
+
def _process_average_emissions(self):
|
|
841
|
+
"""Calculate average emission intensity for all substances and source-types."""
|
|
842
|
+
for substance in self.substances:
|
|
843
|
+
chunk = self._rasterize_average_chunk(substance, self.sourcetypes)
|
|
844
|
+
|
|
845
|
+
if self.unit_conversion_factor != 1.0:
|
|
846
|
+
chunk *= self.unit_conversion_factor
|
|
847
|
+
result_file = os.path.join(self.output.path, substance.slug + ".nc")
|
|
848
|
+
with nc.Dataset(result_file, "a", format="NETCDF4") as dset:
|
|
849
|
+
self.set_data(dset, substance, chunk)
|
|
850
|
+
|
|
851
|
+
def _process_timeseries(self, begin, end): # noqa: C901, PLR0912
|
|
852
|
+
# how many hours that will be processed in the same chunk
|
|
853
|
+
# this is a compromise between required memory, execution time
|
|
854
|
+
# the minimum chunk size of the netcdf variables is used
|
|
855
|
+
|
|
856
|
+
min_time_chunksize = 1e9
|
|
857
|
+
for substance in self.substances:
|
|
858
|
+
result_file = os.path.join(self.output.path, substance.slug + ".nc")
|
|
859
|
+
with nc.Dataset(result_file, "r", format="NETCDF4") as dset:
|
|
860
|
+
# variable_name = 'Emission of '+substance.slug
|
|
861
|
+
# or x, y, time
|
|
862
|
+
time_chunking = dset.variables["time"].chunking()
|
|
863
|
+
if time_chunking[0] < min_time_chunksize:
|
|
864
|
+
min_time_chunksize = time_chunking[0]
|
|
865
|
+
|
|
866
|
+
# rasterizing chunks and writing to dataset
|
|
867
|
+
chunk_begin = begin
|
|
868
|
+
chunk_end = begin
|
|
869
|
+
while chunk_end < end:
|
|
870
|
+
# chunk includes chunk end-time, so nr of hours
|
|
871
|
+
# should be reduced by one to get chunk dimension
|
|
872
|
+
# matching chunksize
|
|
873
|
+
chunk_end = min(
|
|
874
|
+
chunk_begin + datetime.timedelta(hours=min_time_chunksize - 1), end
|
|
875
|
+
)
|
|
876
|
+
self.log.debug(
|
|
877
|
+
f"processing {chunk_begin.strftime('%y%m%d %H')} "
|
|
878
|
+
f"- {chunk_end.strftime('%y%m%d %H')}"
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
# emis intensity scaling time-series are generated from time-profiles
|
|
882
|
+
self._timevariations_to_dataframe(chunk_begin, chunk_end)
|
|
883
|
+
|
|
884
|
+
for substance in self.substances:
|
|
885
|
+
self.log.debug(f"substance: {substance.slug}")
|
|
886
|
+
# add raster timeseries
|
|
887
|
+
emis_chunk = self._rasterize_chunk(
|
|
888
|
+
substance,
|
|
889
|
+
chunk_begin,
|
|
890
|
+
chunk_end,
|
|
891
|
+
self.sourcetypes,
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
if self.unit_conversion_factor != 1.0:
|
|
895
|
+
emis_chunk *= self.unit_conversion_factor
|
|
896
|
+
|
|
897
|
+
result_file = os.path.join(self.output.path, substance.slug + ".nc")
|
|
898
|
+
with nc.Dataset(result_file, "a", format="NETCDF4") as dset:
|
|
899
|
+
self.set_data(
|
|
900
|
+
dset,
|
|
901
|
+
substance,
|
|
902
|
+
emis_chunk,
|
|
903
|
+
timestamps=[chunk_begin, chunk_end],
|
|
904
|
+
)
|
|
905
|
+
# update time-span of variable and dataset
|
|
906
|
+
|
|
907
|
+
# update chunk time interval
|
|
908
|
+
chunk_begin = chunk_end + datetime.timedelta(hours=1)
|
|
909
|
+
|
|
910
|
+
def set_data(self, dset, substance, data, timestamps=None): # noqa: C901, PLR0912
|
|
911
|
+
"""Add chunk of data to variable."""
|
|
912
|
+
|
|
913
|
+
var = dset[f"emission_{substance.slug}"]
|
|
914
|
+
|
|
915
|
+
if timestamps is None:
|
|
916
|
+
# map_oriented, np.flipud flips 2d data along second dimension
|
|
917
|
+
var[:, :] = np.flipud(data)
|
|
918
|
+
else:
|
|
919
|
+
time_var = dset["time"]
|
|
920
|
+
# map_oriented, np.fliplr flips 3d data along second dimension
|
|
921
|
+
var[:, :, :] = np.fliplr(data)
|
|
922
|
+
times = pd.date_range(timestamps[0], timestamps[1], freq=self.time_step)
|
|
923
|
+
hours_since_1970 = (
|
|
924
|
+
times - pd.to_datetime("1970-01-01").tz_localize(self.timezone)
|
|
925
|
+
).total_seconds() / 3600
|
|
926
|
+
time_var[:] = hours_since_1970
|
|
927
|
+
|
|
928
|
+
def _timeseries_emis( # noqa: C901, PLR0912, PLR0915
|
|
929
|
+
self, substance, begin, end, sourcetype
|
|
930
|
+
):
|
|
931
|
+
"""Get emission timeseries data chunk for sources.
|
|
932
|
+
|
|
933
|
+
args:
|
|
934
|
+
substance: substance to be processed as a Parameter instance
|
|
935
|
+
begin: datatime object for first time-step to rasterize
|
|
936
|
+
end: datetime object for last time-step to rasterize
|
|
937
|
+
sourcetype: 'point or 'road'
|
|
938
|
+
"""
|
|
939
|
+
|
|
940
|
+
self.log.debug(f"- discrete {sourcetype}sources")
|
|
941
|
+
# number of timesteps in chunk
|
|
942
|
+
timesteps = int((end - begin).total_seconds() / 3600) + 1
|
|
943
|
+
|
|
944
|
+
# create an empty array
|
|
945
|
+
emis_chunk = np.zeros(
|
|
946
|
+
(timesteps, len(self._cache.feature_ids[sourcetype])), dtype=np.float32
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
# get mapping for query result column index
|
|
950
|
+
col_map = self._cache.col_maps[sourcetype]
|
|
951
|
+
|
|
952
|
+
# a template time-series that will be scaled with emission factors
|
|
953
|
+
emis_fac_freeflow = np.ones((timesteps,))
|
|
954
|
+
|
|
955
|
+
emis_ts = pd.Series(
|
|
956
|
+
index=pd.date_range(start=begin, end=end, freq="h"),
|
|
957
|
+
data=np.zeros((timesteps,)),
|
|
958
|
+
)
|
|
959
|
+
cstvs_cache = {}
|
|
960
|
+
emis_fac_cache = {}
|
|
961
|
+
# iterate over pages
|
|
962
|
+
for page_ind in range(self._cache.emis_page_count(sourcetype, substance.pk)):
|
|
963
|
+
page_nr = page_ind + 1
|
|
964
|
+
try:
|
|
965
|
+
emissions = self._cache.read_emissions(
|
|
966
|
+
sourcetype, substance.pk, page_nr
|
|
967
|
+
)
|
|
968
|
+
except NotInCacheError:
|
|
969
|
+
break
|
|
970
|
+
|
|
971
|
+
# aggregate emissions for each unique feature
|
|
972
|
+
for rec in emissions:
|
|
973
|
+
timevar_id = rec[col_map.timevar_id]
|
|
974
|
+
emis = rec[col_map.emis]
|
|
975
|
+
source_id = rec[col_map.source_id]
|
|
976
|
+
|
|
977
|
+
# for road sources emissions are calculated using
|
|
978
|
+
# ef corresponding to traffic condition for each hour
|
|
979
|
+
if sourcetype == ROAD:
|
|
980
|
+
cstv_id = rec[col_map.coldstart_timevar_id]
|
|
981
|
+
coldstart_fraction = rec[col_map.coldstart_fraction]
|
|
982
|
+
coldstart_ef = rec[col_map.coldstart_ef]
|
|
983
|
+
|
|
984
|
+
# dataframe with coldstart time-variation for each hour
|
|
985
|
+
# (normalize coldstart var combined with flow variation)
|
|
986
|
+
if coldstart_fraction > 0 and coldstart_ef > 0:
|
|
987
|
+
if (cstv_id, timevar_id) not in cstvs_cache:
|
|
988
|
+
cstv_index = pd.date_range(
|
|
989
|
+
start=begin - datetime.timedelta(hours=1),
|
|
990
|
+
end=end - datetime.timedelta(hours=1),
|
|
991
|
+
freq="h",
|
|
992
|
+
)
|
|
993
|
+
cstvs_cache[(cstv_id, timevar_id)] = timevar_to_series(
|
|
994
|
+
cstv_index,
|
|
995
|
+
self.coldstart_timevars[cstv_id or "default"],
|
|
996
|
+
self.flow_timevars[timevar_id or "default"],
|
|
997
|
+
timezone=self.timezone,
|
|
998
|
+
)
|
|
999
|
+
coldstart_timevar_scalings = cstvs_cache[(cstv_id, timevar_id)]
|
|
1000
|
+
|
|
1001
|
+
# dataframe with intensity scalings for each hour
|
|
1002
|
+
flow_timevar_scalings = self.flow_timevar_scalings[
|
|
1003
|
+
timevar_id or "default"
|
|
1004
|
+
]
|
|
1005
|
+
cong_prof_id = rec[col_map.congestion_profile_id or "default"]
|
|
1006
|
+
|
|
1007
|
+
# dataframe with traffic conditions (values 1-4)
|
|
1008
|
+
traffic_conditions = self.traffic_conditions[
|
|
1009
|
+
cong_prof_id or "default"
|
|
1010
|
+
]
|
|
1011
|
+
|
|
1012
|
+
# calculate emission timeseries
|
|
1013
|
+
# exhaust emissions + coldstart emissions
|
|
1014
|
+
flow = rec[col_map.veh_m_per_sec]
|
|
1015
|
+
scaled_veh_flow = flow_timevar_scalings.to_numpy() * flow
|
|
1016
|
+
|
|
1017
|
+
freeflow_ef = rec[col_map.freeflow_ef]
|
|
1018
|
+
heavy_ef = rec[col_map.heavy_ef]
|
|
1019
|
+
saturated_ef = rec[col_map.saturated_ef]
|
|
1020
|
+
stopngo_ef = rec[col_map.stopngo_ef]
|
|
1021
|
+
emis_fac_key = (
|
|
1022
|
+
cong_prof_id,
|
|
1023
|
+
freeflow_ef,
|
|
1024
|
+
heavy_ef,
|
|
1025
|
+
saturated_ef,
|
|
1026
|
+
stopngo_ef,
|
|
1027
|
+
)
|
|
1028
|
+
|
|
1029
|
+
if emis_fac_key not in emis_fac_cache:
|
|
1030
|
+
# array for emission factors for current vehicle
|
|
1031
|
+
# one ef for each traffic condition
|
|
1032
|
+
|
|
1033
|
+
# create a full time-series of free-flow ef
|
|
1034
|
+
emis_fac = emis_fac_freeflow * freeflow_ef
|
|
1035
|
+
|
|
1036
|
+
# replace ef where we have non free-flow conditions
|
|
1037
|
+
emis_fac = np.where(traffic_conditions == 2, heavy_ef, emis_fac)
|
|
1038
|
+
emis_fac = np.where(
|
|
1039
|
+
traffic_conditions == 3, saturated_ef, emis_fac
|
|
1040
|
+
)
|
|
1041
|
+
emis_fac = np.where(
|
|
1042
|
+
traffic_conditions == 4, stopngo_ef, emis_fac
|
|
1043
|
+
)
|
|
1044
|
+
emis_fac_cache[emis_fac_key] = emis_fac
|
|
1045
|
+
else:
|
|
1046
|
+
emis_fac = emis_fac_cache[emis_fac_key]
|
|
1047
|
+
|
|
1048
|
+
# calculate emission timeseries (kg/s)
|
|
1049
|
+
if coldstart_ef > 0 and coldstart_fraction > 0:
|
|
1050
|
+
coldstart_scaled_veh_flow = (
|
|
1051
|
+
coldstart_timevar_scalings.to_numpy()
|
|
1052
|
+
* coldstart_fraction
|
|
1053
|
+
* coldstart_ef
|
|
1054
|
+
)
|
|
1055
|
+
|
|
1056
|
+
emis_ts = (
|
|
1057
|
+
scaled_veh_flow * emis_fac
|
|
1058
|
+
+ flow * coldstart_scaled_veh_flow
|
|
1059
|
+
)
|
|
1060
|
+
else:
|
|
1061
|
+
emis_ts = scaled_veh_flow * emis_fac
|
|
1062
|
+
|
|
1063
|
+
else:
|
|
1064
|
+
# dataframe with intensity scalings for each hour
|
|
1065
|
+
timevar_scalings = self.source_timevar_scalings[
|
|
1066
|
+
timevar_id or "default"
|
|
1067
|
+
]
|
|
1068
|
+
|
|
1069
|
+
# calculate emission timeseries (kg/s)
|
|
1070
|
+
emis_ts = timevar_scalings * emis
|
|
1071
|
+
|
|
1072
|
+
# add emissions to chunk
|
|
1073
|
+
emis_chunk[:, self._cache.feature_ids[sourcetype][source_id]] += emis_ts
|
|
1074
|
+
return emis_chunk
|
|
1075
|
+
|
|
1076
|
+
def _rasterize_average_chunk(self, substance, sourcetypes):
|
|
1077
|
+
"""Rasterize and return emissions as a numpy array.
|
|
1078
|
+
|
|
1079
|
+
args:
|
|
1080
|
+
substance: substance to be processed as Parameter instance
|
|
1081
|
+
sourcetypes: sequence containing 'point', 'area',
|
|
1082
|
+
"""
|
|
1083
|
+
|
|
1084
|
+
# create an empty array
|
|
1085
|
+
chunk = np.zeros((self.ny, self.nx), dtype=np.float32)
|
|
1086
|
+
|
|
1087
|
+
for sourcetype in sourcetypes:
|
|
1088
|
+
if not self._cache.has_sourcetype(sourcetype):
|
|
1089
|
+
continue
|
|
1090
|
+
self.log.debug(f"gridding {sourcetype}sources")
|
|
1091
|
+
# iterate over pages
|
|
1092
|
+
for page_ind in range(
|
|
1093
|
+
self._cache.emis_page_count(sourcetype, substance.pk)
|
|
1094
|
+
):
|
|
1095
|
+
page_nr = page_ind + 1
|
|
1096
|
+
try:
|
|
1097
|
+
weights = self._cache.read_weights(sourcetype, page_nr)
|
|
1098
|
+
emissions = self._cache.read_emissions(
|
|
1099
|
+
sourcetype, substance.pk, page_nr
|
|
1100
|
+
)
|
|
1101
|
+
except NotInCacheError:
|
|
1102
|
+
break
|
|
1103
|
+
|
|
1104
|
+
for rec in emissions:
|
|
1105
|
+
col_map = self._cache.col_maps[sourcetype]
|
|
1106
|
+
emis = rec[col_map.emis]
|
|
1107
|
+
source_id = rec[col_map.source_id]
|
|
1108
|
+
|
|
1109
|
+
# NOTE: time-series emissions will not be possible to include
|
|
1110
|
+
# without specifying a time-interval.
|
|
1111
|
+
if sourcetype != GRID:
|
|
1112
|
+
# get cell indices and weights for source
|
|
1113
|
+
index_array, source_weights = weights[source_id]
|
|
1114
|
+
else:
|
|
1115
|
+
# get cell indices and weights for grid source raster
|
|
1116
|
+
raster_name = rec[col_map.raster]
|
|
1117
|
+
# get cell indices and weights for source
|
|
1118
|
+
try:
|
|
1119
|
+
index_array, source_weights = weights[
|
|
1120
|
+
(source_id, raster_name)
|
|
1121
|
+
]
|
|
1122
|
+
except KeyError:
|
|
1123
|
+
# grid has no emissions within modelling area
|
|
1124
|
+
continue
|
|
1125
|
+
|
|
1126
|
+
# add emissions to chunk at each index
|
|
1127
|
+
chunk[index_array] += emis * source_weights
|
|
1128
|
+
|
|
1129
|
+
return chunk
|
|
1130
|
+
|
|
1131
|
+
def _timeseries_average_chunk(self, substance, sourcetype):
|
|
1132
|
+
"""Get emissions for sources as a numpy array.
|
|
1133
|
+
|
|
1134
|
+
args:
|
|
1135
|
+
substance: substance to be processed as Parameter instance
|
|
1136
|
+
sourcetype: 'point' or 'road'
|
|
1137
|
+
"""
|
|
1138
|
+
|
|
1139
|
+
# create an empty array
|
|
1140
|
+
chunk = np.zeros((self.ny, self.nx), dtype=np.float32)
|
|
1141
|
+
|
|
1142
|
+
# create an empty array
|
|
1143
|
+
chunk = np.zeros((len(self._cache.feature_ids[sourcetype]),), dtype=np.float32)
|
|
1144
|
+
|
|
1145
|
+
col_map = self._cache.col_maps[sourcetype]
|
|
1146
|
+
# iterate over pages
|
|
1147
|
+
for page_ind in range(self._cache.emis_page_count(sourcetype, substance.pk)):
|
|
1148
|
+
page_nr = page_ind + 1
|
|
1149
|
+
try:
|
|
1150
|
+
emissions = self._cache.read_emissions(
|
|
1151
|
+
sourcetype, substance.pk, page_nr
|
|
1152
|
+
)
|
|
1153
|
+
except NotInCacheError:
|
|
1154
|
+
break
|
|
1155
|
+
|
|
1156
|
+
# aggregate emissions for each unique feature
|
|
1157
|
+
for rec in emissions:
|
|
1158
|
+
emis = rec[col_map.emis]
|
|
1159
|
+
source_id = rec[col_map.source_id]
|
|
1160
|
+
chunk[self._cache.feature_ids[sourcetype][source_id]] += emis
|
|
1161
|
+
return chunk
|
|
1162
|
+
|
|
1163
|
+
def _rasterize_chunk( # noqa: C901, PLR0912, PLR0915
|
|
1164
|
+
self, substance, begin, end, sourcetypes
|
|
1165
|
+
):
|
|
1166
|
+
"""Rasterize and return as numpy array.
|
|
1167
|
+
|
|
1168
|
+
args:
|
|
1169
|
+
substance: substance to process as Parameter instance
|
|
1170
|
+
begin: datatime object for first time-step to rasterize
|
|
1171
|
+
end: datetime object for last time-step to rasterize
|
|
1172
|
+
sourcetypes: sequence containing 'point', 'area'
|
|
1173
|
+
"""
|
|
1174
|
+
|
|
1175
|
+
# number of timesteps in chunk
|
|
1176
|
+
nr_timesteps = int((end - begin).total_seconds() / 3600) + 1
|
|
1177
|
+
|
|
1178
|
+
# create an empty array
|
|
1179
|
+
chunk = np.zeros((nr_timesteps, self.ny, self.nx), dtype=np.float32)
|
|
1180
|
+
emis_fac_freeflow = np.ones((nr_timesteps,))
|
|
1181
|
+
cstvs_cache = {}
|
|
1182
|
+
emis_fac_cache = {}
|
|
1183
|
+
for sourcetype in sourcetypes:
|
|
1184
|
+
if not self._cache.has_sourcetype(sourcetype):
|
|
1185
|
+
continue
|
|
1186
|
+
self.log.debug(f"-gridding {sourcetype}sources")
|
|
1187
|
+
# iterate over pages
|
|
1188
|
+
for page_ind in range(
|
|
1189
|
+
self._cache.emis_page_count(sourcetype, substance.pk)
|
|
1190
|
+
):
|
|
1191
|
+
page_nr = page_ind + 1
|
|
1192
|
+
try:
|
|
1193
|
+
weights = self._cache.read_weights(sourcetype, page_nr)
|
|
1194
|
+
emissions = self._cache.read_emissions(
|
|
1195
|
+
sourcetype, substance.pk, page_nr
|
|
1196
|
+
)
|
|
1197
|
+
except NotInCacheError:
|
|
1198
|
+
break
|
|
1199
|
+
|
|
1200
|
+
for rec in emissions:
|
|
1201
|
+
col_map = self._cache.col_maps[sourcetype]
|
|
1202
|
+
timevar_id = rec[col_map.timevar_id]
|
|
1203
|
+
emis = rec[col_map.emis]
|
|
1204
|
+
source_id = rec[col_map.source_id]
|
|
1205
|
+
|
|
1206
|
+
# for road sources emissions are calculated using
|
|
1207
|
+
# ef corresponding to traffic condition for each hour
|
|
1208
|
+
if sourcetype == ROAD:
|
|
1209
|
+
cstv_id = rec[col_map.coldstart_timevar_id]
|
|
1210
|
+
coldstart_fraction = rec[col_map.coldstart_fraction]
|
|
1211
|
+
coldstart_ef = rec[col_map.coldstart_ef]
|
|
1212
|
+
|
|
1213
|
+
# dataframe with coldstart time-variation for each hour
|
|
1214
|
+
# (normalize coldstart var combined with flow variation)
|
|
1215
|
+
if coldstart_fraction > 0 and coldstart_ef > 0:
|
|
1216
|
+
if (cstv_id, timevar_id) not in cstvs_cache:
|
|
1217
|
+
cstv_index = pd.date_range(
|
|
1218
|
+
start=begin - datetime.timedelta(hours=1),
|
|
1219
|
+
end=end - datetime.timedelta(hours=1),
|
|
1220
|
+
freq="h",
|
|
1221
|
+
)
|
|
1222
|
+
cstvs_cache[(cstv_id, timevar_id)] = timevar_to_series(
|
|
1223
|
+
cstv_index,
|
|
1224
|
+
self.coldstart_timevars[cstv_id or "default"],
|
|
1225
|
+
self.flow_timevars[timevar_id or "default"],
|
|
1226
|
+
timezone=self.timezone,
|
|
1227
|
+
)
|
|
1228
|
+
coldstart_timevar_scalings = cstvs_cache[
|
|
1229
|
+
(cstv_id, timevar_id)
|
|
1230
|
+
]
|
|
1231
|
+
|
|
1232
|
+
# dataframe with intensity scalings for each hour
|
|
1233
|
+
flow_timevar_scalings = self.flow_timevar_scalings[
|
|
1234
|
+
timevar_id or "default"
|
|
1235
|
+
]
|
|
1236
|
+
cong_prof_id = rec[col_map.congestion_profile_id or "default"]
|
|
1237
|
+
|
|
1238
|
+
# dataframe with traffic conditions (values 1-4)
|
|
1239
|
+
traffic_conditions = self.traffic_conditions[
|
|
1240
|
+
cong_prof_id or "default"
|
|
1241
|
+
]
|
|
1242
|
+
|
|
1243
|
+
# calculate emission timeseries
|
|
1244
|
+
# exhaust emissions + coldstart emissions
|
|
1245
|
+
flow = rec[col_map.veh_m_per_sec]
|
|
1246
|
+
scaled_veh_flow = flow_timevar_scalings.to_numpy() * flow
|
|
1247
|
+
|
|
1248
|
+
freeflow_ef = rec[col_map.freeflow_ef]
|
|
1249
|
+
heavy_ef = rec[col_map.heavy_ef]
|
|
1250
|
+
saturated_ef = rec[col_map.saturated_ef]
|
|
1251
|
+
stopngo_ef = rec[col_map.stopngo_ef]
|
|
1252
|
+
|
|
1253
|
+
emis_fac_key = (
|
|
1254
|
+
cong_prof_id,
|
|
1255
|
+
freeflow_ef,
|
|
1256
|
+
heavy_ef,
|
|
1257
|
+
saturated_ef,
|
|
1258
|
+
stopngo_ef,
|
|
1259
|
+
)
|
|
1260
|
+
if emis_fac_key not in emis_fac_cache:
|
|
1261
|
+
# array for emission factors for current vehicle
|
|
1262
|
+
# one ef for each traffic condition
|
|
1263
|
+
emis_fac = emis_fac_freeflow * freeflow_ef
|
|
1264
|
+
emis_fac = np.where(
|
|
1265
|
+
traffic_conditions == 2, heavy_ef, emis_fac
|
|
1266
|
+
)
|
|
1267
|
+
emis_fac = np.where(
|
|
1268
|
+
traffic_conditions == 3, saturated_ef, emis_fac
|
|
1269
|
+
)
|
|
1270
|
+
emis_fac = np.where(
|
|
1271
|
+
traffic_conditions == 4, stopngo_ef, emis_fac
|
|
1272
|
+
)
|
|
1273
|
+
emis_fac_cache[emis_fac_key] = emis_fac
|
|
1274
|
+
else:
|
|
1275
|
+
emis_fac = emis_fac_cache[emis_fac_key]
|
|
1276
|
+
|
|
1277
|
+
if coldstart_ef > 0 and coldstart_fraction > 0:
|
|
1278
|
+
coldstart_scaled_veh_flow = (
|
|
1279
|
+
coldstart_timevar_scalings.to_numpy()
|
|
1280
|
+
* coldstart_fraction
|
|
1281
|
+
* coldstart_ef
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
emis_ts = (
|
|
1285
|
+
scaled_veh_flow * emis_fac
|
|
1286
|
+
+ flow * coldstart_scaled_veh_flow
|
|
1287
|
+
)
|
|
1288
|
+
else:
|
|
1289
|
+
emis_ts = scaled_veh_flow * emis_fac
|
|
1290
|
+
|
|
1291
|
+
else:
|
|
1292
|
+
# dataframe with intensity scalings for each hour
|
|
1293
|
+
timevar_scalings = self.source_timevar_scalings[
|
|
1294
|
+
timevar_id or "default"
|
|
1295
|
+
]
|
|
1296
|
+
|
|
1297
|
+
# calculate emission timeseries scaled by timevar
|
|
1298
|
+
emis_ts = timevar_scalings.to_numpy() * emis
|
|
1299
|
+
|
|
1300
|
+
if sourcetype != GRID:
|
|
1301
|
+
# get cell indices and weights for source
|
|
1302
|
+
index_array, source_weights = weights[source_id]
|
|
1303
|
+
else:
|
|
1304
|
+
raster_name = rec[col_map.raster]
|
|
1305
|
+
try:
|
|
1306
|
+
index_array, source_weights = weights[
|
|
1307
|
+
(source_id, raster_name)
|
|
1308
|
+
]
|
|
1309
|
+
except KeyError:
|
|
1310
|
+
# grid has no emissions within modelling area
|
|
1311
|
+
continue
|
|
1312
|
+
# add time dimension to index array
|
|
1313
|
+
try:
|
|
1314
|
+
ncells = index_array[0].size
|
|
1315
|
+
except AttributeError:
|
|
1316
|
+
ncells = len(index_array[0])
|
|
1317
|
+
index_array = (
|
|
1318
|
+
np.tile(np.arange(nr_timesteps), ncells),
|
|
1319
|
+
np.repeat(index_array[0], nr_timesteps),
|
|
1320
|
+
np.repeat(index_array[1], nr_timesteps),
|
|
1321
|
+
)
|
|
1322
|
+
|
|
1323
|
+
# organize 1D-array of emissions as the index array
|
|
1324
|
+
emis_array = (source_weights * emis_ts[:, np.newaxis]).T.reshape(-1)
|
|
1325
|
+
|
|
1326
|
+
# add emissions to chunk at each index
|
|
1327
|
+
chunk[index_array] += emis_array
|
|
1328
|
+
return chunk
|
|
1329
|
+
|
|
1330
|
+
def _calc_chunking(
|
|
1331
|
+
self, time_chunksize=None, spatial_chunksize=None, chunk_cache=1e7
|
|
1332
|
+
):
|
|
1333
|
+
"""Return chunksizes for variable dimensions."""
|
|
1334
|
+
|
|
1335
|
+
# if there is no time-dimension, chunking is not needed
|
|
1336
|
+
# for grids smaller than 100 x 100, chunksize covers the whole grid
|
|
1337
|
+
if len(self.timevars) == 0:
|
|
1338
|
+
return None
|
|
1339
|
+
|
|
1340
|
+
# ensure chunk size is not larger than grid
|
|
1341
|
+
chunk_nx = min(self.nx, spatial_chunksize or 5)
|
|
1342
|
+
chunk_ny = min(self.ny, spatial_chunksize or 5)
|
|
1343
|
+
|
|
1344
|
+
if time_chunksize is None:
|
|
1345
|
+
# assumes single precision (4 bytes per value)
|
|
1346
|
+
# max estimate based on full grid rather than limiting to spatial chunks,
|
|
1347
|
+
# (more practical if full grid fits in memory...)
|
|
1348
|
+
|
|
1349
|
+
field_size_bytes = self.nx * self.ny * 4
|
|
1350
|
+
max_time_chunksize = int(chunk_cache / field_size_bytes)
|
|
1351
|
+
|
|
1352
|
+
if self.time_step is not None:
|
|
1353
|
+
time_chunksize = int(
|
|
1354
|
+
datetime.timedelta(hours=366 * 24)
|
|
1355
|
+
/ time_step_as_delta(self.time_step)
|
|
1356
|
+
)
|
|
1357
|
+
else:
|
|
1358
|
+
time_chunksize = 366
|
|
1359
|
+
|
|
1360
|
+
time_chunksize = min(time_chunksize, max_time_chunksize)
|
|
1361
|
+
|
|
1362
|
+
return (time_chunksize, chunk_ny, chunk_nx)
|
|
1363
|
+
|
|
1364
|
+
def create_gridmapping_variable(self, dset: nc.Dataset, crs: rio.CRS):
|
|
1365
|
+
name = f"EPSG_{crs.to_epsg()}"
|
|
1366
|
+
grid_mapping = dset.createVariable(name, "i")
|
|
1367
|
+
epsg = crs.to_epsg()
|
|
1368
|
+
if epsg is None:
|
|
1369
|
+
epsg = crs.to_epsg(confidence_threshold=30)
|
|
1370
|
+
self.log.info(
|
|
1371
|
+
f"No exact match for EPSG and spatial ref, using best guess: {epsg}"
|
|
1372
|
+
)
|
|
1373
|
+
grid_mapping.srid = epsg
|
|
1374
|
+
grid_mapping.crs_wkt = crs.to_wkt() # for CF Conventions 1.7
|
|
1375
|
+
return grid_mapping
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def create_variable(
|
|
1379
|
+
dset: nc.Dataset,
|
|
1380
|
+
grid_mapping: nc.Variable,
|
|
1381
|
+
name: str,
|
|
1382
|
+
unit: Optional[str] = None,
|
|
1383
|
+
quantity: Optional[str] = None,
|
|
1384
|
+
instance: Optional[str] = None,
|
|
1385
|
+
cell_methods: Optional[str] = None,
|
|
1386
|
+
parameter: Optional[str] = None,
|
|
1387
|
+
height: Optional[float] = None,
|
|
1388
|
+
time: bool = True,
|
|
1389
|
+
chunksizes: Optional[tuple[int, ...]] = None,
|
|
1390
|
+
):
|
|
1391
|
+
"""create 2D variable."""
|
|
1392
|
+
if time:
|
|
1393
|
+
var = dset.createVariable(
|
|
1394
|
+
name,
|
|
1395
|
+
"f4",
|
|
1396
|
+
(TIME_NAME, Y_NAME, X_NAME),
|
|
1397
|
+
fill_value=np.nan,
|
|
1398
|
+
zlib=True,
|
|
1399
|
+
chunksizes=chunksizes,
|
|
1400
|
+
)
|
|
1401
|
+
else:
|
|
1402
|
+
var = dset.createVariable(
|
|
1403
|
+
name, "f4", (Y_NAME, X_NAME), fill_value=np.nan, zlib=True
|
|
1404
|
+
)
|
|
1405
|
+
|
|
1406
|
+
var.units = unit or "unknown"
|
|
1407
|
+
var.grid_mapping = grid_mapping.name
|
|
1408
|
+
if parameter is not None:
|
|
1409
|
+
var.parameter = parameter
|
|
1410
|
+
if quantity is not None:
|
|
1411
|
+
var.quantity = quantity
|
|
1412
|
+
if instance is not None:
|
|
1413
|
+
var.instance = instance
|
|
1414
|
+
if cell_methods is not None:
|
|
1415
|
+
var.cell_methods = cell_methods
|
|
1416
|
+
if height is not None:
|
|
1417
|
+
var.height = height
|
|
1418
|
+
return var
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
GENERAL_ATTRIBUTES = [("Created_using_cetk", __version__)]
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def write_general_attrs(dset: nc.Dataset):
|
|
1425
|
+
"""add some global attributes."""
|
|
1426
|
+
for name, attr in GENERAL_ATTRIBUTES:
|
|
1427
|
+
if not hasattr(dset, name):
|
|
1428
|
+
setattr(dset, name, attr)
|
|
1429
|
+
|
|
1430
|
+
if not hasattr(dset, "history"):
|
|
1431
|
+
timestamp = datetime.datetime.now(tz=datetime.timezone.utc).strftime(
|
|
1432
|
+
"%Y%m%d %H:%M"
|
|
1433
|
+
)
|
|
1434
|
+
dset.history = f"{timestamp} created dataset"
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
# name of time-dimension and time-variable
|
|
1438
|
+
TIME_NAME = "time"
|
|
1439
|
+
X_NAME = "x"
|
|
1440
|
+
Y_NAME = "y"
|
|
1441
|
+
# name of time-bounds dimension and time-bounds variable
|
|
1442
|
+
TIME_BOUNDS_NAME = "time_bounds"
|
|
1443
|
+
# name of second dimension for bounds variables
|
|
1444
|
+
BOUNDS_DIMENSION = "bounds_dim"
|
|
1445
|
+
# netCDF reference time
|
|
1446
|
+
TIME_ZERO = datetime.datetime(
|
|
1447
|
+
year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=datetime.timezone.utc
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
|
|
1451
|
+
def create_time_variable(dset: nc.Dataset):
|
|
1452
|
+
"""create time variable."""
|
|
1453
|
+
|
|
1454
|
+
times = dset.createDimension(TIME_NAME, None)
|
|
1455
|
+
dset.createDimension(BOUNDS_DIMENSION, 2)
|
|
1456
|
+
times = dset.createVariable(TIME_NAME, "f8", (TIME_NAME,))
|
|
1457
|
+
times.calendar = "gregorian"
|
|
1458
|
+
times.units = f"hours since {TIME_ZERO:%Y-%m-%d %H:%M:%S}"
|
|
1459
|
+
times.bounds = TIME_BOUNDS_NAME
|
|
1460
|
+
times.long_name = "time"
|
|
1461
|
+
times.axis = "T"
|
|
1462
|
+
time_bounds = dset.createVariable(
|
|
1463
|
+
TIME_BOUNDS_NAME, "f8", (TIME_NAME, BOUNDS_DIMENSION)
|
|
1464
|
+
)
|
|
1465
|
+
return times, time_bounds
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
def create_xy_variables(
|
|
1469
|
+
dset: nc.Dataset, extent: tuple, crs: rio.CRS, nx: int, ny: int
|
|
1470
|
+
):
|
|
1471
|
+
"""Create a netCDF dataset with dimensions and metadata for 2D fields."""
|
|
1472
|
+
|
|
1473
|
+
x1, y1, x2, y2 = extent
|
|
1474
|
+
dx = (x2 - x1) / nx
|
|
1475
|
+
dy = (y2 - y1) / ny
|
|
1476
|
+
|
|
1477
|
+
dset.createDimension(X_NAME, nx)
|
|
1478
|
+
# create projected coordinate variables
|
|
1479
|
+
x = dset.createVariable(X_NAME, "f4", (X_NAME,))
|
|
1480
|
+
x.standard_name = "projection_x_coordinate"
|
|
1481
|
+
x.units = "m"
|
|
1482
|
+
x.axis = "X"
|
|
1483
|
+
x.description = "center of cell"
|
|
1484
|
+
x[:] = np.linspace(x1 + 0.5 * dx, x2 - 0.5 * dx, nx)
|
|
1485
|
+
|
|
1486
|
+
dset.createDimension(Y_NAME, ny)
|
|
1487
|
+
y = dset.createVariable(Y_NAME, "f4", (Y_NAME,))
|
|
1488
|
+
y.standard_name = "projection_y_coordinate"
|
|
1489
|
+
y.units = "m"
|
|
1490
|
+
y.axis = "Y"
|
|
1491
|
+
y.description = "center of cell"
|
|
1492
|
+
y[:] = np.linspace(y1 + 0.5 * dy, y2 - 0.5 * dy, ny)
|
|
1493
|
+
return x, y
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
def make_naive_in_utc(time: datetime.datetime):
|
|
1497
|
+
"""convert time to utc and make naive."""
|
|
1498
|
+
if time.tzinfo is not None:
|
|
1499
|
+
time_utc = time.astimezone(datetime.timezone.utc)
|
|
1500
|
+
time_utc.replace(tzinfo=None)
|
|
1501
|
+
else:
|
|
1502
|
+
time_utc = time
|
|
1503
|
+
return time_utc
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def write_time(
|
|
1507
|
+
time_var: nc.Variable, time_bounds_var: nc.Variable, time_utc: datetime.datetime
|
|
1508
|
+
):
|
|
1509
|
+
"""write timestamp to file and return time-index"""
|
|
1510
|
+
|
|
1511
|
+
left_bound = time_utc - datetime.timedelta(hours=1)
|
|
1512
|
+
# get index of time in netCDF
|
|
1513
|
+
if len(time_var) == 0:
|
|
1514
|
+
time_index = 0
|
|
1515
|
+
else:
|
|
1516
|
+
time_index = nc.date2index(time_utc, time_var, select="before") + 1
|
|
1517
|
+
|
|
1518
|
+
# write time to netCDF (in numerical format)
|
|
1519
|
+
nc_time = nc.date2num(time_utc, time_var.units)
|
|
1520
|
+
nc_left_time_bound = nc.date2num(left_bound, time_var.units)
|
|
1521
|
+
nc_right_time_bound = nc_time
|
|
1522
|
+
|
|
1523
|
+
time_bounds_var[time_index, :] = [nc_left_time_bound, nc_right_time_bound]
|
|
1524
|
+
time_var[time_index] = nc_time
|
|
1525
|
+
return time_index
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
def get_nodes_from_wkt(wkt):
|
|
1529
|
+
"""Extract list of nodes (x, y) from WKT.
|
|
1530
|
+
|
|
1531
|
+
geom_type: 'POINT', 'LINESTRING' or 'POLYGON'
|
|
1532
|
+
|
|
1533
|
+
"""
|
|
1534
|
+
geom = GEOSGeometry(wkt)
|
|
1535
|
+
if geom.geom_type == "Point":
|
|
1536
|
+
return (geom.coords,)
|
|
1537
|
+
if geom.geom_type == "LineString":
|
|
1538
|
+
return geom.coords
|
|
1539
|
+
if geom.geom_type == "Polygon":
|
|
1540
|
+
return geom.coords[0]
|
|
1541
|
+
|
|
1542
|
+
raise NotImplementedError(
|
|
1543
|
+
f'Geometry type specified in WKT not implemented: "{wkt}"'
|
|
1544
|
+
)
|