cetk 0.0.4.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. cetk/__init__.py +87 -0
  2. cetk/db.py +41 -0
  3. cetk/edb/__init__.py +0 -0
  4. cetk/edb/apps.py +9 -0
  5. cetk/edb/cache.py +249 -0
  6. cetk/edb/const.py +26 -0
  7. cetk/edb/copy.py +18 -0
  8. cetk/edb/exporters.py +551 -0
  9. cetk/edb/importers/__init__.py +36 -0
  10. cetk/edb/importers/activity_import.py +279 -0
  11. cetk/edb/importers/codeset_import.py +124 -0
  12. cetk/edb/importers/gridsource_import.py +256 -0
  13. cetk/edb/importers/roadsource_import.py +1602 -0
  14. cetk/edb/importers/source_import.py +958 -0
  15. cetk/edb/importers/timevar_import.py +115 -0
  16. cetk/edb/importers/utils.py +92 -0
  17. cetk/edb/importers/validation.py +207 -0
  18. cetk/edb/ltreefield.py +76 -0
  19. cetk/edb/migrations/0001_initial.py +1731 -0
  20. cetk/edb/migrations/0002_data.py +156 -0
  21. cetk/edb/migrations/0003_create_gpkg_tables.py +14 -0
  22. cetk/edb/migrations/__init__.py +0 -0
  23. cetk/edb/models/__init__.py +49 -0
  24. cetk/edb/models/base.py +52 -0
  25. cetk/edb/models/common_models.py +144 -0
  26. cetk/edb/models/fleets.py +160 -0
  27. cetk/edb/models/gridsource_models.py +163 -0
  28. cetk/edb/models/road_classes.py +197 -0
  29. cetk/edb/models/road_models.py +413 -0
  30. cetk/edb/models/source_models.py +399 -0
  31. cetk/edb/models/timevar_models.py +111 -0
  32. cetk/edb/rasterize/__init__.py +1 -0
  33. cetk/edb/rasterize/rasterizer.py +1544 -0
  34. cetk/edb/signals.py +43 -0
  35. cetk/edb/traffic.py +75 -0
  36. cetk/edb/units.py +160 -0
  37. cetk/emissions/__init__.py +0 -0
  38. cetk/emissions/aggregate_emissions.sql +303 -0
  39. cetk/emissions/areasource_emissions.sql +53 -0
  40. cetk/emissions/calc.py +148 -0
  41. cetk/emissions/filters.py +179 -0
  42. cetk/emissions/gridsource_emissions.sql +52 -0
  43. cetk/emissions/pointsource_emissions.sql +53 -0
  44. cetk/emissions/queries.py +221 -0
  45. cetk/emissions/roadsource_emissions.sql +211 -0
  46. cetk/emissions/used_substances.sql +56 -0
  47. cetk/emissions/views.py +89 -0
  48. cetk/logging.py +68 -0
  49. cetk/settings.py +85 -0
  50. cetk/tools/__init__.py +1 -0
  51. cetk/tools/cetk_command.py +599 -0
  52. cetk/tools/manage.py +24 -0
  53. cetk/tools/utils.py +334 -0
  54. cetk/utils.py +62 -0
  55. cetk-0.0.4.dev0.dist-info/LICENSE +201 -0
  56. cetk-0.0.4.dev0.dist-info/METADATA +297 -0
  57. cetk-0.0.4.dev0.dist-info/RECORD +60 -0
  58. cetk-0.0.4.dev0.dist-info/WHEEL +5 -0
  59. cetk-0.0.4.dev0.dist-info/entry_points.txt +3 -0
  60. cetk-0.0.4.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1544 @@
1
+ """Rasterization of emission sources."""
2
+
3
+ import datetime
4
+ import logging
5
+ import os
6
+ from copy import copy
7
+ from functools import cache
8
+ from itertools import product
9
+ from math import ceil
10
+ from pathlib import Path
11
+ from typing import Optional
12
+
13
+ import netCDF4 as nc
14
+ import numpy as np
15
+ import pandas as pd
16
+ import rasterio as rio
17
+ from django.contrib.gis.geos import GEOSGeometry, Polygon
18
+ from rastafari import ddaf_line_subpixel, even_odd_polygon_fill, resample_band
19
+
20
+ from cetk import __version__
21
+ from cetk.edb.cache import EmissionCache, NotInCacheError
22
+ from cetk.edb.models import (
23
+ VELOCITY_CHOICES,
24
+ AreaSource,
25
+ ColdstartTimevar,
26
+ CongestionProfile,
27
+ FlowTimevar,
28
+ GridSource,
29
+ Parameter,
30
+ PointSource,
31
+ RoadSource,
32
+ Settings,
33
+ Timevar,
34
+ get_gridsource_raster,
35
+ timevar_to_series,
36
+ )
37
+ from cetk.edb.models.gridsource_models import OutsideExtentError
38
+ from cetk.edb.traffic import los_to_velocity
39
+ from cetk.edb.units import emis_conversion_factor_from_si
40
+ from cetk.emissions.calc import calculate_source_emissions
41
+
42
+ # short source type identifiers for conveniency
43
+ POINT = PointSource.sourcetype
44
+ AREA = AreaSource.sourcetype
45
+ GRID = GridSource.sourcetype
46
+ ROAD = RoadSource.sourcetype
47
+
48
+ # supported source types
49
+ SOURCETYPES = (POINT, AREA, GRID, ROAD)
50
+
51
+ # slug for traffic work parameter and traffic condition parameter
52
+ TRAFFIC_WORK_PARAMETER_SLUG = "traffic_work"
53
+ TRAFFIC_CONDITION_PARAMETER_SLUG = "traffic_condition"
54
+ TRAFFIC_WORK_SUBSTANCE_SLUG = "traffic_work"
55
+ VEHICLE_SPEED_PARAMETER_SLUG = "vehicle_speed"
56
+
57
+
58
+ class Output:
59
+ def __init__(
60
+ self, extent: tuple, path: Path, srid: int, timezone: datetime.timezone
61
+ ):
62
+ self.extent = extent
63
+ self.path = path
64
+ self.srid = srid
65
+ self.timezone = timezone
66
+
67
+
68
+ @cache
69
+ def time_step_as_delta(time_step):
70
+ """Return timedelta corresponding to a time_step.
71
+ For Y and M, returns an approximate time-delta.
72
+ """
73
+
74
+ if time_step.endswith("Y"):
75
+ nyears = int(time_step[:-1])
76
+ return pd.Timedelta(days=nyears * 365)
77
+
78
+ if time_step.endswith("M"):
79
+ nmonths = int(time_step[:-1])
80
+ return pd.Timedelta(days=nmonths * 30)
81
+
82
+ ts = time_step.lower() if time_step.endswith("h") else time_step
83
+ try:
84
+ return pd.Timedelta(ts)
85
+ except ValueError:
86
+ raise ValueError(f"invalid time step abbreviation {ts}")
87
+
88
+
89
+ class EmissionRasterizer:
90
+ """An emission time-series rasterizer.
91
+
92
+ writes emissions to outputfile in units kg/s
93
+
94
+ """
95
+
96
+ def __init__(self, output, nx, ny):
97
+ """
98
+ args
99
+ output: output.path: path to store NetCDF files
100
+ output.extent: extent of the raster (x1, y1, x2, y2)
101
+ output.srid: SRID of output, as int of 4 numbers (ex. 3006)
102
+ output.timezone: timezone of output
103
+ nx: raster dimension in x-direction
104
+ ny: raster dimension in y-direction
105
+ """
106
+ self.timevars = {}
107
+ self.flow_timevars = {}
108
+ self.coldstart_timevars = {}
109
+ self.congestion_profiles = {}
110
+ self.level_weights = {}
111
+ self.output = output
112
+ self.extent = output.extent
113
+ self.crs = rio.crs.CRS.from_epsg(output.srid)
114
+ self._cache = None
115
+ self.nx = nx
116
+ self.ny = ny
117
+ self.sourcetypes = []
118
+
119
+ # querysets for emission
120
+ self.querysets = {}
121
+
122
+ self.log = logging.getLogger(__name__)
123
+
124
+ def reset(self):
125
+ """Reset attributes.
126
+ To allow reuse of rasterizer (e.g. for a new substance)
127
+ """
128
+
129
+ self.querysets = {}
130
+ self.sourcetypes = []
131
+
132
+ @property
133
+ def x1(self):
134
+ return self.extent[0]
135
+
136
+ @property
137
+ def x2(self):
138
+ return self.extent[2]
139
+
140
+ @property
141
+ def y1(self):
142
+ return self.extent[1]
143
+
144
+ @property
145
+ def y2(self):
146
+ return self.extent[3]
147
+
148
+ @property
149
+ def dx(self):
150
+ return (self.x2 - self.x1) / self.nx
151
+
152
+ @property
153
+ def dy(self):
154
+ return (self.y2 - self.y1) / self.ny
155
+
156
+ @property
157
+ def srid(self):
158
+ return self.output.srid
159
+
160
+ def _get_timevars(self, sourcetypes):
161
+ """Get all time-variations."""
162
+ self.timevars = {}
163
+ self.flow_timevars = {}
164
+ self.coldstart_timevars = {}
165
+ self.congestion_profiles = {}
166
+
167
+ if POINT in sourcetypes or AREA in sourcetypes:
168
+ for tvar in Timevar.objects.all():
169
+ self.timevars[tvar.id] = tvar
170
+ default = Timevar()
171
+ self.timevars["default"] = default
172
+
173
+ if ROAD in sourcetypes:
174
+ for tvar in FlowTimevar.objects.all():
175
+ self.flow_timevars[tvar.id] = tvar
176
+ default = FlowTimevar()
177
+ self.flow_timevars["default"] = default
178
+
179
+ for tvar in ColdstartTimevar.objects.all():
180
+ self.coldstart_timevars[tvar.id] = tvar
181
+ default = ColdstartTimevar()
182
+ self.coldstart_timevars["default"] = default
183
+
184
+ for prof in CongestionProfile.objects.all():
185
+ self.congestion_profiles[prof.id] = prof
186
+ default = CongestionProfile()
187
+ self.congestion_profiles["default"] = default
188
+
189
+ def _get_level_weights(self):
190
+ """redistribute vertical distr. into levels of result dataset."""
191
+ # TODO don't use levels yet
192
+ if Settings.get_current().codeset1 is None:
193
+ self.log.warning(
194
+ "inventory has no primary code-set - "
195
+ "cannot apply vertical distributions "
196
+ )
197
+ return
198
+
199
+ bounds = self.levels[:]
200
+
201
+ weights = np.zeros(self.levels.shape, dtype=float)
202
+ # cache vertical distributions to used for different activitycodes
203
+ # heights are determined using the 1st (primary) codeset
204
+ for ac in (
205
+ Settings.get_current()
206
+ .codeset1.codes.prefetch_related("vertical_dist")
207
+ .exclude(vertical_dist__isnull=True)
208
+ ):
209
+ # eval is needed in cetk because arrays stored as CharField in sqlite
210
+ vdist = np.array(eval(ac.vertical_dist.weights))
211
+ vdist_weights = vdist[:, 1]
212
+ vdist_heights = vdist[:, 0]
213
+
214
+ if vdist_heights[0] > 0:
215
+ vdist_weights = np.insert(vdist_weights, 0, 0)
216
+ vdist_heights = np.insert(vdist_heights, 0, 0)
217
+
218
+ # interp weights at level bounds
219
+ interp_weights = np.interp(bounds, vdist_heights, vdist_weights, right=0)
220
+ # insert interpolated values into vdist
221
+ start_ind = 0
222
+ level = 0
223
+ for bound, val in zip(bounds, interp_weights):
224
+ # index where bound will be inserted
225
+ ind = np.searchsorted(vdist_heights, bound)
226
+ if bound not in vdist_heights:
227
+ vdist_weights = np.insert(vdist_weights, ind, val)
228
+ vdist_heights = np.insert(vdist_heights, ind, bound)
229
+ # integrate weight between level bounds
230
+ weight = np.trapz(
231
+ vdist_weights[start_ind : ind + 1],
232
+ vdist_heights[start_ind : ind + 1],
233
+ )
234
+ start_ind = ind
235
+ weights[level] = weight
236
+ level += 1
237
+ weights_sum = weights.sum()
238
+ if weights_sum != 0:
239
+ weights /= weights_sum
240
+ vertical_index = weights.nonzero()
241
+ non_zero_weights = weights[vertical_index]
242
+ self.level_weights[ac.code] = (vertical_index[0], non_zero_weights[:])
243
+
244
+ def _timevariations_to_dataframe(self, begin, end):
245
+ """Prepare hourly dataframes in UTC for all time-variation profiles.
246
+
247
+ args
248
+ begin: start time in UTC
249
+ end: end time in UTC
250
+
251
+ """
252
+ time_index = pd.date_range(begin, end, freq="h")
253
+ one_hour = datetime.timedelta(hours=1)
254
+ shifted_index = pd.date_range(begin - one_hour, end - one_hour, freq="h")
255
+
256
+ self.flow_timevar_scalings = pd.DataFrame(
257
+ {
258
+ fvar.id
259
+ or "default": timevar_to_series(
260
+ shifted_index, fvar, timezone=self.timezone
261
+ ).to_numpy() # discard the shifted index by converting to ndarray
262
+ for fvar in self.flow_timevars.values()
263
+ },
264
+ index=time_index,
265
+ )
266
+ self.traffic_conditions = pd.DataFrame(
267
+ {
268
+ prof.id
269
+ or "default": prof.to_series(
270
+ shifted_index, timezone=self.timezone
271
+ ).to_numpy()
272
+ for prof in self.congestion_profiles.values()
273
+ },
274
+ index=time_index,
275
+ )
276
+
277
+ # calculate vehicle velocity from level-of-service and posted speed
278
+ self.velocity_light = {}
279
+ self.velocity_heavy = {}
280
+ for key, vel in product(self.traffic_conditions, VELOCITY_CHOICES):
281
+ lvehspd, hvehspd = los_to_velocity(self.traffic_conditions[key], vel)
282
+ # convert to SI units, km/h to m/s
283
+ self.velocity_light[(key, vel)] = lvehspd / 3.6
284
+ self.velocity_heavy[(key, vel)] = hvehspd / 3.6
285
+
286
+ self.source_timevar_scalings = pd.DataFrame(
287
+ {
288
+ tvar.id
289
+ or "default": timevar_to_series(
290
+ shifted_index, tvar, timezone=self.output.timezone
291
+ ).to_numpy()
292
+ for tvar in self.timevars.values()
293
+ },
294
+ index=time_index,
295
+ )
296
+
297
+ def _get_querysets( # noqa: C901, PLR0912
298
+ self,
299
+ sourcetypes=None,
300
+ name=None,
301
+ tags=None,
302
+ point_ids=None,
303
+ area_ids=None,
304
+ grid_ids=None,
305
+ road_ids=None,
306
+ polygon=None,
307
+ substances=None,
308
+ ac1=None,
309
+ ac2=None,
310
+ ac3=None,
311
+ *,
312
+ cur=None,
313
+ ):
314
+ """Get querysets for emissions."""
315
+
316
+ sourcetypes = sourcetypes or SOURCETYPES
317
+ # not used; srid = Settings.get_current().srid
318
+
319
+ if ac1 is not None or ac2 is not None or ac3 is not None:
320
+ raise ValueError(
321
+ "filtering for ac not implemented in source_emissions_query yet!"
322
+ )
323
+ # if activity codes are given as strings, get the corresponding
324
+ # activity code instances
325
+ if ac1 is not None and len(ac1) > 0 and isinstance(ac1[0], str):
326
+ # TODO why code__in= not code= ?
327
+ ac1_instances = list(
328
+ Settings.get_current().codeset1.codes.filter(code__in=ac1)
329
+ )
330
+ if len(ac1_instances) == 0:
331
+ raise ValueError(
332
+ f"the filter for activitycode1: {ac1} does not match any code"
333
+ f" in code-set '{Settings.get_current().codeset1.name}'"
334
+ )
335
+ else:
336
+ ac1_instances = ac1
337
+
338
+ if ac2 is not None and len(ac2) > 0 and isinstance(ac2[0], str):
339
+ ac2_instances = list(
340
+ Settings.get_current().codeset2.codes.filter(code__in=ac2)
341
+ )
342
+ if len(ac2_instances) == 0:
343
+ raise ValueError(
344
+ f"the filter for activitycode2: {ac2} does not match any code"
345
+ f" in code-set '{Settings.get_current().codeset2.name}'"
346
+ )
347
+ else:
348
+ ac2_instances = ac2
349
+
350
+ if ac3 is not None and len(ac3) > 0 and isinstance(ac3[0], str):
351
+ ac3_instances = list(
352
+ Settings.get_current().codeset3.codes.filter(code__in=ac3)
353
+ )
354
+ if len(ac3_instances) == 0:
355
+ raise ValueError(
356
+ f"the filter for activitycode3: {ac3} does not match any code"
357
+ f" in code-set '{Settings.get_current().codeset3.name}'"
358
+ )
359
+ else:
360
+ ac3_instances = ac3
361
+
362
+ # aggregate emissions for each source-type per sourceid
363
+ # TODO, ac1-3 and tolerance not used in calculate_source_emissions yet
364
+ # tolerance = None
365
+ for sourcetype in sourcetypes:
366
+ self.log.debug(f"- {sourcetype}source emissions")
367
+ if sourcetype == POINT:
368
+ if not PointSource.objects.exists():
369
+ continue
370
+ ids = point_ids
371
+ elif sourcetype == AREA:
372
+ if not AreaSource.objects.exists():
373
+ continue
374
+ ids = area_ids
375
+ elif sourcetype == ROAD:
376
+ if not RoadSource.objects.exists():
377
+ continue
378
+ ids = road_ids
379
+ elif sourcetype == GRID:
380
+ if not GridSource.objects.exists():
381
+ continue
382
+ ids = grid_ids
383
+ else:
384
+ raise ValueError(
385
+ f"Rasterize cannot handle sourcetype {sourcetype} yet."
386
+ )
387
+ self.querysets[sourcetype] = calculate_source_emissions(
388
+ sourcetype,
389
+ substances=substances,
390
+ srid=int(self.output.srid),
391
+ name=name,
392
+ ids=ids,
393
+ tags=tags,
394
+ polygon=polygon,
395
+ )
396
+
397
+ def _get_weights(self, polygon=None):
398
+ """Get cell weights for all sourcetypes."""
399
+
400
+ for sourcetype in self.querysets:
401
+ if sourcetype == POINT:
402
+ self._get_point_weights()
403
+ elif sourcetype == AREA:
404
+ self._get_area_weights()
405
+ if sourcetype == ROAD:
406
+ self._get_road_weights()
407
+ elif sourcetype == GRID:
408
+ self._get_grid_weights(polygon=polygon)
409
+
410
+ def _get_point_weights(self):
411
+ """Get cell weights for points.
412
+ returns a dict with source_id as keys and
413
+ (index_array, 1.0) as values
414
+
415
+ complete records from query result are stored in dict
416
+ with source_id as key.
417
+ """
418
+ col_map = self._cache.col_maps[POINT]
419
+ for rec in self.querysets[POINT]:
420
+ source_id = rec[col_map.source_id]
421
+
422
+ if source_id not in self._cache.gridded_sources[POINT]:
423
+ [[x, y]] = np.array(get_nodes_from_wkt(rec[col_map.wkt]))
424
+
425
+ # check if source is within bounds
426
+ if (x < self.x1 or y < self.y1) or (x > self.x2 or y > self.y2):
427
+ continue
428
+
429
+ col = min(int((x - self.x1) / self.dx), self.nx - 1)
430
+ row = min(self.ny - int(ceil((y - self.y1) / self.dy)), self.ny - 1)
431
+ index = ((row,), (col,))
432
+
433
+ # stores index array for consistency with other sourcetypes
434
+ weights = (index, 1.0)
435
+ self._cache.add_rec(rec, POINT, write_weights=True, weights=weights)
436
+ else:
437
+ self._cache.add_rec(rec, POINT, write_weights=False)
438
+
439
+ # write last page to cache
440
+ self._cache.write_weights(POINT)
441
+ self._cache.write_emissions(POINT)
442
+
443
+ def _get_area_weights(self):
444
+ """Get cell weights for area sources.
445
+ returns a dict with source_id as keys and
446
+ (index_array, weight_array) as values
447
+
448
+ complete records from query result are stored in a dict
449
+ with source_id as key.
450
+ """
451
+
452
+ col_map = self._cache.col_maps[AREA]
453
+ for rec in self.querysets[AREA]:
454
+ source_id = rec[col_map.source_id]
455
+
456
+ if source_id not in self._cache.gridded_sources[AREA]:
457
+ source_weights = {}
458
+ # extract nodes from area geometry in WKT format
459
+ wkt = rec[col_map.wkt]
460
+ nodes = np.array(get_nodes_from_wkt(wkt))
461
+
462
+ is_within_extent = any(
463
+ self.x1 <= x <= self.x2 and self.y1 <= y <= self.y2
464
+ for x, y in nodes
465
+ )
466
+
467
+ if is_within_extent:
468
+ even_odd_polygon_fill(
469
+ nodes,
470
+ source_weights,
471
+ self.extent,
472
+ self.nx,
473
+ self.ny,
474
+ subgridcells=2,
475
+ )
476
+
477
+ if len(source_weights) > 0:
478
+ indices = list(zip(*source_weights.keys()))
479
+
480
+ index = (indices[0], indices[1])
481
+
482
+ weights = (
483
+ index,
484
+ np.fromiter(source_weights.values(), dtype=float),
485
+ )
486
+ else:
487
+ weights = None
488
+ self._cache.add_rec(rec, AREA, write_weights=True, weights=weights)
489
+ else:
490
+ self._cache.add_rec(rec, AREA, write_weights=False)
491
+
492
+ # write last cache page to disk
493
+ self._cache.write_weights(AREA)
494
+ self._cache.write_emissions(AREA)
495
+
496
+ def _get_road_weights(self):
497
+ """Get cell weights for roads.
498
+ returns a dict with road_id as keys and
499
+ (index_array, weight_array) as values
500
+
501
+ complete records from query result are stored in dict
502
+ with road_id as key.
503
+ """
504
+
505
+ col_map = self._cache.col_maps[ROAD]
506
+ for rec in self.querysets[ROAD]:
507
+ source_id = rec[col_map.source_id]
508
+ if source_id not in self._cache.gridded_sources[ROAD]:
509
+ road_weights = {}
510
+ # extract nodes from road geometry in WKT format
511
+ wkt = rec[col_map.wkt]
512
+ nodes = get_nodes_from_wkt(wkt)
513
+
514
+ # if geometry is simplified, the length of the simplified
515
+ # geometry should be used when rasterizing
516
+ try:
517
+ road_length = rec[col_map.simple_length]
518
+ except AttributeError:
519
+ road_length = rec[col_map.length]
520
+
521
+ # sum weights for each segment to each cell index
522
+ # (many segments can intersect the same cell)
523
+
524
+ for i in range(1, len(nodes)):
525
+ p1_x, p1_y = nodes[i - 1]
526
+ p2_x, p2_y = nodes[i]
527
+
528
+ # rasterize segment and add weights to road_weights dict
529
+ # for each segment the weights are multiplied with
530
+ # the segment fraction of the road
531
+ # the resulting road weights will sum up to 1
532
+ ddaf_line_subpixel(
533
+ p1_x,
534
+ p1_y,
535
+ p2_x,
536
+ p2_y,
537
+ road_weights,
538
+ road_length,
539
+ self.extent,
540
+ self.dx,
541
+ self.dy,
542
+ )
543
+
544
+ if len(road_weights) > 0:
545
+ indices = list(zip(*road_weights.keys()))
546
+
547
+ try:
548
+ ncells = indices[0].size
549
+ except AttributeError:
550
+ ncells = len(indices[0])
551
+ if self.levels is not None:
552
+ # road sources are added to lowest level
553
+ level = 0
554
+ # add z dimension to index array
555
+ index = (tuple([level] * ncells), indices[0], indices[1])
556
+ else:
557
+ index = (indices[0], indices[1])
558
+
559
+ # store cell indices and a weight array for each road id
560
+ weights = (
561
+ index,
562
+ np.fromiter(road_weights.values(), dtype=float),
563
+ )
564
+ else:
565
+ weights = None
566
+ self._cache.add_rec(rec, ROAD, write_weights=True, weights=weights)
567
+ else:
568
+ self._cache.add_rec(rec, ROAD, write_weights=False)
569
+
570
+ # write last cache page to disk
571
+ self._cache.write_weights(ROAD)
572
+ # write a separate file for records of each substance
573
+ self._cache.write_emissions(ROAD)
574
+
575
+ def _get_grid_weights(self, polygon=None):
576
+ """Store records of grid-sources in dict with source_id as key.
577
+
578
+ returns a dict with source_id as keys and
579
+ (index_array, weight_array) as values.
580
+
581
+ Complete records from query
582
+ result are stored in a dict with source_id as key.
583
+ """
584
+ col_map = self._cache.col_maps[GRID]
585
+ for rec in self.querysets[GRID]:
586
+ source_id = rec[col_map.source_id]
587
+ raster_name = rec[col_map.raster]
588
+ # spatial distribution of a grid is unique for each raster
589
+ source_key = (source_id, raster_name)
590
+ if source_key not in self._cache.gridded_sources[GRID]:
591
+ try:
592
+ raster_data, metadata = get_gridsource_raster(
593
+ raster_name, clip_by=polygon
594
+ )
595
+ except OutsideExtentError:
596
+ # no overlap between query and this grid,
597
+ # try next gridsource
598
+ break
599
+
600
+ index_array, source_weights = resample_band(
601
+ raster_data,
602
+ metadata["extent"],
603
+ metadata["nodata"],
604
+ self.extent,
605
+ self.nx,
606
+ self.ny,
607
+ metadata["srid"],
608
+ self.srid,
609
+ subgridcells=2,
610
+ )
611
+
612
+ if len(source_weights) > 0:
613
+ if self.levels is not None:
614
+ # add weights in vertical direction
615
+ ac1 = rec[col_map.ac1]
616
+
617
+ if ac1 is not None:
618
+ # add indices and weights for vertical levels
619
+ try:
620
+ levels, level_weights = self.level_weights[ac1]
621
+ except KeyError:
622
+ # if no vertical distribution is assigned to the
623
+ # activity code, all emissions are put in the
624
+ # lowest layer
625
+ levels = np.array([0])
626
+ level_weights = np.ones(1, dtype=float)
627
+ index_array = (
628
+ np.tile(levels, index_array[0].size),
629
+ np.repeat(index_array[0], levels.size),
630
+ np.repeat(index_array[1], levels.size),
631
+ )
632
+ # weight each level with corresponding level-weight
633
+ source_weights = source_weights.repeat(
634
+ level_weights.size
635
+ ) * np.tile(level_weights, source_weights.size)
636
+ else:
637
+ index_array = (
638
+ np.zeros(index_array.size[0]),
639
+ index_array[0],
640
+ index_array[1],
641
+ )
642
+ weights = (index_array, source_weights)
643
+ else:
644
+ weights = None
645
+
646
+ # store emission record
647
+ self._cache.add_rec(
648
+ rec,
649
+ GRID,
650
+ write_weights=True,
651
+ weights=weights,
652
+ )
653
+ else:
654
+ self._cache.add_rec(rec, GRID)
655
+
656
+ # write last cache page to disk
657
+ self._cache.write_weights(GRID)
658
+ self._cache.write_emissions(GRID)
659
+
660
+ def _create_variables( # noqa: C901, PLR0912, PLR0915
661
+ self, substances, *, timeseries=True
662
+ ):
663
+ """create netCDF variables."""
664
+ if timeseries:
665
+ cell_methods = "1h-mean"
666
+ time = True
667
+ self.time_step = "1h"
668
+ else:
669
+ cell_methods = None
670
+ time = False
671
+ self.time_step = None
672
+
673
+ self.variables = {}
674
+ for substance in substances:
675
+ result_file = os.path.join(self.output.path, substance.slug + ".nc")
676
+ with nc.Dataset(result_file, "w", format="NETCDF4") as dset:
677
+ write_general_attrs(dset)
678
+ grid_mapping_var = self.create_gridmapping_variable(dset, self.crs)
679
+ time_var, time_bounds_var = create_time_variable(dset)
680
+ create_xy_variables(dset, self.extent, self.crs, self.nx, self.ny)
681
+ subst_vars = self.variables.setdefault(substance.slug, {})
682
+ # create variables also without emission in extent
683
+ # if any(
684
+ # self._cache.has_substance(sourcetype, substance.id)
685
+ # for sourcetype in self.sourcetypes
686
+ # ):
687
+ param = Parameter.objects.get(quantity="emission", substance=substance)
688
+ chunking = self._calc_chunking(
689
+ chunk_cache=1e8,
690
+ )
691
+ var_name = param.slug
692
+ create_variable(
693
+ dset,
694
+ grid_mapping_var,
695
+ name=var_name,
696
+ unit=self.unit,
697
+ instance=self.instance,
698
+ cell_methods=cell_methods,
699
+ parameter=param.name,
700
+ time=time,
701
+ chunksizes=chunking,
702
+ )
703
+ subst_vars["field2d"] = {"emission": var_name}
704
+ ncreated = 0
705
+ for subst_vars in self.variables.values():
706
+ for sourcetype_vars in subst_vars.values():
707
+ ncreated += len(sourcetype_vars)
708
+ return ncreated > 0
709
+
710
+ def process( # noqa: PLR0915
711
+ self,
712
+ substances,
713
+ begin=None,
714
+ end=None,
715
+ sourcetypes=None,
716
+ exclude_points=False,
717
+ exclude_roads=False,
718
+ point_ids=None,
719
+ area_ids=None,
720
+ grid_ids=None,
721
+ road_ids=None,
722
+ name=None,
723
+ tags=None,
724
+ polygon=None,
725
+ ac1=None,
726
+ ac2=None,
727
+ ac3=None,
728
+ cur=None,
729
+ unit=None,
730
+ aggregation=None,
731
+ instance=None,
732
+ ):
733
+ """Write emissions to dataset.
734
+
735
+ args:
736
+ substances: an iterable of (or single) Substance model instances
737
+
738
+ optional args:
739
+ begin: datetime object in utc for first time rasterize
740
+ end: datetime object in utc for last time to rasterize
741
+ sourcetypes: 'point', 'area', 'grid' or 'road'
742
+ exclude_points: point source emissions not rasterized
743
+ exclude_roads: road source emissions not rasterized
744
+ road_ids: list of road id's to include
745
+ point_ids: list of road id's to include
746
+ area_ids: list of road id's to include
747
+ grid_ids: list of road id's to include
748
+ name: source name (accepts regexp)
749
+ tags: dictionary with tags and values to filter on
750
+ polygon: only include sources within the provided polygon
751
+ ac1: iterable of activity codes
752
+ ac2: iterable of activity codes
753
+ ac3: iterable of activity codes
754
+ cur: db cursor object, used to execute query if provided
755
+ unit: emission output unit
756
+ aggregation: aggregation label of result variables
757
+ instance: write emissions to specified variable instance
758
+ """
759
+ self.instance = instance
760
+ self.aggregation = aggregation
761
+ self.levels = None
762
+ if polygon is None:
763
+ # output extent
764
+ x1, y1, x2, y2 = self.extent
765
+ polygon = Polygon(
766
+ ((x1, y1), (x2, y1), (x2, y2), (x1, y2), (x1, y1)),
767
+ srid=int(self.output.srid),
768
+ )
769
+ self.unit = unit or "kg/s"
770
+ self.unit_conversion_factor = emis_conversion_factor_from_si(self.unit)
771
+
772
+ # determine which sourcetypes should be rasterized
773
+ self.sourcetypes = copy(sourcetypes) or list(SOURCETYPES)
774
+ if exclude_points and POINT in self.sourcetypes:
775
+ self.sourcetypes.remove(POINT)
776
+ if exclude_roads and ROAD in self.sourcetypes:
777
+ self.sourcetypes.remove(ROAD)
778
+
779
+ if not hasattr(substances, "__iter__"):
780
+ self.substances = [substances]
781
+ else:
782
+ self.substances = substances
783
+
784
+ self.log.debug("reading records from database")
785
+
786
+ # create querysets for all sources
787
+ self._get_querysets(
788
+ sourcetypes=sourcetypes,
789
+ name=name,
790
+ tags=tags,
791
+ point_ids=point_ids,
792
+ area_ids=area_ids,
793
+ road_ids=road_ids,
794
+ grid_ids=grid_ids,
795
+ polygon=polygon,
796
+ ac1=ac1,
797
+ ac2=ac2,
798
+ ac3=ac3,
799
+ cur=cur,
800
+ )
801
+ try:
802
+ Path(self.output.path).mkdir(exist_ok=True)
803
+ except FileNotFoundError:
804
+ raise FileNotFoundError(
805
+ "parent directory of rasterized output does not exist "
806
+ f"({str(self.output.path)})"
807
+ )
808
+
809
+ with EmissionCache(self.querysets) as cache:
810
+ self._cache = cache
811
+
812
+ self.log.debug("processing static data")
813
+ # calculate raster cell weights for sources in querysets
814
+ # and store in a dict
815
+ self._get_weights(polygon=polygon)
816
+
817
+ self.timezone = self.output.timezone
818
+
819
+ if begin is not None and end is not None:
820
+ self.log.debug("creating result variables")
821
+ created = self._create_variables(self.substances, timeseries=True)
822
+
823
+ # get time-variation profiles
824
+ self._get_timevars(sourcetypes or SOURCETYPES)
825
+ self.log.debug("calculating time-series emissions")
826
+ self._process_timeseries(begin, end)
827
+ else:
828
+ self.log.debug("creating result variables")
829
+ created = self._create_variables(self.substances, timeseries=False)
830
+ # if no variables are created, skip out
831
+ if not created:
832
+ self.reset()
833
+ self.log.info("no emissions found within requested extent")
834
+ return
835
+ self.log.debug("calculating average emissions")
836
+ self._process_average_emissions()
837
+
838
+ self.reset()
839
+
840
+ def _process_average_emissions(self):
841
+ """Calculate average emission intensity for all substances and source-types."""
842
+ for substance in self.substances:
843
+ chunk = self._rasterize_average_chunk(substance, self.sourcetypes)
844
+
845
+ if self.unit_conversion_factor != 1.0:
846
+ chunk *= self.unit_conversion_factor
847
+ result_file = os.path.join(self.output.path, substance.slug + ".nc")
848
+ with nc.Dataset(result_file, "a", format="NETCDF4") as dset:
849
+ self.set_data(dset, substance, chunk)
850
+
851
+ def _process_timeseries(self, begin, end): # noqa: C901, PLR0912
852
+ # how many hours that will be processed in the same chunk
853
+ # this is a compromise between required memory, execution time
854
+ # the minimum chunk size of the netcdf variables is used
855
+
856
+ min_time_chunksize = 1e9
857
+ for substance in self.substances:
858
+ result_file = os.path.join(self.output.path, substance.slug + ".nc")
859
+ with nc.Dataset(result_file, "r", format="NETCDF4") as dset:
860
+ # variable_name = 'Emission of '+substance.slug
861
+ # or x, y, time
862
+ time_chunking = dset.variables["time"].chunking()
863
+ if time_chunking[0] < min_time_chunksize:
864
+ min_time_chunksize = time_chunking[0]
865
+
866
+ # rasterizing chunks and writing to dataset
867
+ chunk_begin = begin
868
+ chunk_end = begin
869
+ while chunk_end < end:
870
+ # chunk includes chunk end-time, so nr of hours
871
+ # should be reduced by one to get chunk dimension
872
+ # matching chunksize
873
+ chunk_end = min(
874
+ chunk_begin + datetime.timedelta(hours=min_time_chunksize - 1), end
875
+ )
876
+ self.log.debug(
877
+ f"processing {chunk_begin.strftime('%y%m%d %H')} "
878
+ f"- {chunk_end.strftime('%y%m%d %H')}"
879
+ )
880
+
881
+ # emis intensity scaling time-series are generated from time-profiles
882
+ self._timevariations_to_dataframe(chunk_begin, chunk_end)
883
+
884
+ for substance in self.substances:
885
+ self.log.debug(f"substance: {substance.slug}")
886
+ # add raster timeseries
887
+ emis_chunk = self._rasterize_chunk(
888
+ substance,
889
+ chunk_begin,
890
+ chunk_end,
891
+ self.sourcetypes,
892
+ )
893
+
894
+ if self.unit_conversion_factor != 1.0:
895
+ emis_chunk *= self.unit_conversion_factor
896
+
897
+ result_file = os.path.join(self.output.path, substance.slug + ".nc")
898
+ with nc.Dataset(result_file, "a", format="NETCDF4") as dset:
899
+ self.set_data(
900
+ dset,
901
+ substance,
902
+ emis_chunk,
903
+ timestamps=[chunk_begin, chunk_end],
904
+ )
905
+ # update time-span of variable and dataset
906
+
907
+ # update chunk time interval
908
+ chunk_begin = chunk_end + datetime.timedelta(hours=1)
909
+
910
+ def set_data(self, dset, substance, data, timestamps=None): # noqa: C901, PLR0912
911
+ """Add chunk of data to variable."""
912
+
913
+ var = dset[f"emission_{substance.slug}"]
914
+
915
+ if timestamps is None:
916
+ # map_oriented, np.flipud flips 2d data along second dimension
917
+ var[:, :] = np.flipud(data)
918
+ else:
919
+ time_var = dset["time"]
920
+ # map_oriented, np.fliplr flips 3d data along second dimension
921
+ var[:, :, :] = np.fliplr(data)
922
+ times = pd.date_range(timestamps[0], timestamps[1], freq=self.time_step)
923
+ hours_since_1970 = (
924
+ times - pd.to_datetime("1970-01-01").tz_localize(self.timezone)
925
+ ).total_seconds() / 3600
926
+ time_var[:] = hours_since_1970
927
+
928
+ def _timeseries_emis( # noqa: C901, PLR0912, PLR0915
929
+ self, substance, begin, end, sourcetype
930
+ ):
931
+ """Get emission timeseries data chunk for sources.
932
+
933
+ args:
934
+ substance: substance to be processed as a Parameter instance
935
+ begin: datatime object for first time-step to rasterize
936
+ end: datetime object for last time-step to rasterize
937
+ sourcetype: 'point or 'road'
938
+ """
939
+
940
+ self.log.debug(f"- discrete {sourcetype}sources")
941
+ # number of timesteps in chunk
942
+ timesteps = int((end - begin).total_seconds() / 3600) + 1
943
+
944
+ # create an empty array
945
+ emis_chunk = np.zeros(
946
+ (timesteps, len(self._cache.feature_ids[sourcetype])), dtype=np.float32
947
+ )
948
+
949
+ # get mapping for query result column index
950
+ col_map = self._cache.col_maps[sourcetype]
951
+
952
+ # a template time-series that will be scaled with emission factors
953
+ emis_fac_freeflow = np.ones((timesteps,))
954
+
955
+ emis_ts = pd.Series(
956
+ index=pd.date_range(start=begin, end=end, freq="h"),
957
+ data=np.zeros((timesteps,)),
958
+ )
959
+ cstvs_cache = {}
960
+ emis_fac_cache = {}
961
+ # iterate over pages
962
+ for page_ind in range(self._cache.emis_page_count(sourcetype, substance.pk)):
963
+ page_nr = page_ind + 1
964
+ try:
965
+ emissions = self._cache.read_emissions(
966
+ sourcetype, substance.pk, page_nr
967
+ )
968
+ except NotInCacheError:
969
+ break
970
+
971
+ # aggregate emissions for each unique feature
972
+ for rec in emissions:
973
+ timevar_id = rec[col_map.timevar_id]
974
+ emis = rec[col_map.emis]
975
+ source_id = rec[col_map.source_id]
976
+
977
+ # for road sources emissions are calculated using
978
+ # ef corresponding to traffic condition for each hour
979
+ if sourcetype == ROAD:
980
+ cstv_id = rec[col_map.coldstart_timevar_id]
981
+ coldstart_fraction = rec[col_map.coldstart_fraction]
982
+ coldstart_ef = rec[col_map.coldstart_ef]
983
+
984
+ # dataframe with coldstart time-variation for each hour
985
+ # (normalize coldstart var combined with flow variation)
986
+ if coldstart_fraction > 0 and coldstart_ef > 0:
987
+ if (cstv_id, timevar_id) not in cstvs_cache:
988
+ cstv_index = pd.date_range(
989
+ start=begin - datetime.timedelta(hours=1),
990
+ end=end - datetime.timedelta(hours=1),
991
+ freq="h",
992
+ )
993
+ cstvs_cache[(cstv_id, timevar_id)] = timevar_to_series(
994
+ cstv_index,
995
+ self.coldstart_timevars[cstv_id or "default"],
996
+ self.flow_timevars[timevar_id or "default"],
997
+ timezone=self.timezone,
998
+ )
999
+ coldstart_timevar_scalings = cstvs_cache[(cstv_id, timevar_id)]
1000
+
1001
+ # dataframe with intensity scalings for each hour
1002
+ flow_timevar_scalings = self.flow_timevar_scalings[
1003
+ timevar_id or "default"
1004
+ ]
1005
+ cong_prof_id = rec[col_map.congestion_profile_id or "default"]
1006
+
1007
+ # dataframe with traffic conditions (values 1-4)
1008
+ traffic_conditions = self.traffic_conditions[
1009
+ cong_prof_id or "default"
1010
+ ]
1011
+
1012
+ # calculate emission timeseries
1013
+ # exhaust emissions + coldstart emissions
1014
+ flow = rec[col_map.veh_m_per_sec]
1015
+ scaled_veh_flow = flow_timevar_scalings.to_numpy() * flow
1016
+
1017
+ freeflow_ef = rec[col_map.freeflow_ef]
1018
+ heavy_ef = rec[col_map.heavy_ef]
1019
+ saturated_ef = rec[col_map.saturated_ef]
1020
+ stopngo_ef = rec[col_map.stopngo_ef]
1021
+ emis_fac_key = (
1022
+ cong_prof_id,
1023
+ freeflow_ef,
1024
+ heavy_ef,
1025
+ saturated_ef,
1026
+ stopngo_ef,
1027
+ )
1028
+
1029
+ if emis_fac_key not in emis_fac_cache:
1030
+ # array for emission factors for current vehicle
1031
+ # one ef for each traffic condition
1032
+
1033
+ # create a full time-series of free-flow ef
1034
+ emis_fac = emis_fac_freeflow * freeflow_ef
1035
+
1036
+ # replace ef where we have non free-flow conditions
1037
+ emis_fac = np.where(traffic_conditions == 2, heavy_ef, emis_fac)
1038
+ emis_fac = np.where(
1039
+ traffic_conditions == 3, saturated_ef, emis_fac
1040
+ )
1041
+ emis_fac = np.where(
1042
+ traffic_conditions == 4, stopngo_ef, emis_fac
1043
+ )
1044
+ emis_fac_cache[emis_fac_key] = emis_fac
1045
+ else:
1046
+ emis_fac = emis_fac_cache[emis_fac_key]
1047
+
1048
+ # calculate emission timeseries (kg/s)
1049
+ if coldstart_ef > 0 and coldstart_fraction > 0:
1050
+ coldstart_scaled_veh_flow = (
1051
+ coldstart_timevar_scalings.to_numpy()
1052
+ * coldstart_fraction
1053
+ * coldstart_ef
1054
+ )
1055
+
1056
+ emis_ts = (
1057
+ scaled_veh_flow * emis_fac
1058
+ + flow * coldstart_scaled_veh_flow
1059
+ )
1060
+ else:
1061
+ emis_ts = scaled_veh_flow * emis_fac
1062
+
1063
+ else:
1064
+ # dataframe with intensity scalings for each hour
1065
+ timevar_scalings = self.source_timevar_scalings[
1066
+ timevar_id or "default"
1067
+ ]
1068
+
1069
+ # calculate emission timeseries (kg/s)
1070
+ emis_ts = timevar_scalings * emis
1071
+
1072
+ # add emissions to chunk
1073
+ emis_chunk[:, self._cache.feature_ids[sourcetype][source_id]] += emis_ts
1074
+ return emis_chunk
1075
+
1076
+ def _rasterize_average_chunk(self, substance, sourcetypes):
1077
+ """Rasterize and return emissions as a numpy array.
1078
+
1079
+ args:
1080
+ substance: substance to be processed as Parameter instance
1081
+ sourcetypes: sequence containing 'point', 'area',
1082
+ """
1083
+
1084
+ # create an empty array
1085
+ chunk = np.zeros((self.ny, self.nx), dtype=np.float32)
1086
+
1087
+ for sourcetype in sourcetypes:
1088
+ if not self._cache.has_sourcetype(sourcetype):
1089
+ continue
1090
+ self.log.debug(f"gridding {sourcetype}sources")
1091
+ # iterate over pages
1092
+ for page_ind in range(
1093
+ self._cache.emis_page_count(sourcetype, substance.pk)
1094
+ ):
1095
+ page_nr = page_ind + 1
1096
+ try:
1097
+ weights = self._cache.read_weights(sourcetype, page_nr)
1098
+ emissions = self._cache.read_emissions(
1099
+ sourcetype, substance.pk, page_nr
1100
+ )
1101
+ except NotInCacheError:
1102
+ break
1103
+
1104
+ for rec in emissions:
1105
+ col_map = self._cache.col_maps[sourcetype]
1106
+ emis = rec[col_map.emis]
1107
+ source_id = rec[col_map.source_id]
1108
+
1109
+ # NOTE: time-series emissions will not be possible to include
1110
+ # without specifying a time-interval.
1111
+ if sourcetype != GRID:
1112
+ # get cell indices and weights for source
1113
+ index_array, source_weights = weights[source_id]
1114
+ else:
1115
+ # get cell indices and weights for grid source raster
1116
+ raster_name = rec[col_map.raster]
1117
+ # get cell indices and weights for source
1118
+ try:
1119
+ index_array, source_weights = weights[
1120
+ (source_id, raster_name)
1121
+ ]
1122
+ except KeyError:
1123
+ # grid has no emissions within modelling area
1124
+ continue
1125
+
1126
+ # add emissions to chunk at each index
1127
+ chunk[index_array] += emis * source_weights
1128
+
1129
+ return chunk
1130
+
1131
+ def _timeseries_average_chunk(self, substance, sourcetype):
1132
+ """Get emissions for sources as a numpy array.
1133
+
1134
+ args:
1135
+ substance: substance to be processed as Parameter instance
1136
+ sourcetype: 'point' or 'road'
1137
+ """
1138
+
1139
+ # create an empty array
1140
+ chunk = np.zeros((self.ny, self.nx), dtype=np.float32)
1141
+
1142
+ # create an empty array
1143
+ chunk = np.zeros((len(self._cache.feature_ids[sourcetype]),), dtype=np.float32)
1144
+
1145
+ col_map = self._cache.col_maps[sourcetype]
1146
+ # iterate over pages
1147
+ for page_ind in range(self._cache.emis_page_count(sourcetype, substance.pk)):
1148
+ page_nr = page_ind + 1
1149
+ try:
1150
+ emissions = self._cache.read_emissions(
1151
+ sourcetype, substance.pk, page_nr
1152
+ )
1153
+ except NotInCacheError:
1154
+ break
1155
+
1156
+ # aggregate emissions for each unique feature
1157
+ for rec in emissions:
1158
+ emis = rec[col_map.emis]
1159
+ source_id = rec[col_map.source_id]
1160
+ chunk[self._cache.feature_ids[sourcetype][source_id]] += emis
1161
+ return chunk
1162
+
1163
+ def _rasterize_chunk( # noqa: C901, PLR0912, PLR0915
1164
+ self, substance, begin, end, sourcetypes
1165
+ ):
1166
+ """Rasterize and return as numpy array.
1167
+
1168
+ args:
1169
+ substance: substance to process as Parameter instance
1170
+ begin: datatime object for first time-step to rasterize
1171
+ end: datetime object for last time-step to rasterize
1172
+ sourcetypes: sequence containing 'point', 'area'
1173
+ """
1174
+
1175
+ # number of timesteps in chunk
1176
+ nr_timesteps = int((end - begin).total_seconds() / 3600) + 1
1177
+
1178
+ # create an empty array
1179
+ chunk = np.zeros((nr_timesteps, self.ny, self.nx), dtype=np.float32)
1180
+ emis_fac_freeflow = np.ones((nr_timesteps,))
1181
+ cstvs_cache = {}
1182
+ emis_fac_cache = {}
1183
+ for sourcetype in sourcetypes:
1184
+ if not self._cache.has_sourcetype(sourcetype):
1185
+ continue
1186
+ self.log.debug(f"-gridding {sourcetype}sources")
1187
+ # iterate over pages
1188
+ for page_ind in range(
1189
+ self._cache.emis_page_count(sourcetype, substance.pk)
1190
+ ):
1191
+ page_nr = page_ind + 1
1192
+ try:
1193
+ weights = self._cache.read_weights(sourcetype, page_nr)
1194
+ emissions = self._cache.read_emissions(
1195
+ sourcetype, substance.pk, page_nr
1196
+ )
1197
+ except NotInCacheError:
1198
+ break
1199
+
1200
+ for rec in emissions:
1201
+ col_map = self._cache.col_maps[sourcetype]
1202
+ timevar_id = rec[col_map.timevar_id]
1203
+ emis = rec[col_map.emis]
1204
+ source_id = rec[col_map.source_id]
1205
+
1206
+ # for road sources emissions are calculated using
1207
+ # ef corresponding to traffic condition for each hour
1208
+ if sourcetype == ROAD:
1209
+ cstv_id = rec[col_map.coldstart_timevar_id]
1210
+ coldstart_fraction = rec[col_map.coldstart_fraction]
1211
+ coldstart_ef = rec[col_map.coldstart_ef]
1212
+
1213
+ # dataframe with coldstart time-variation for each hour
1214
+ # (normalize coldstart var combined with flow variation)
1215
+ if coldstart_fraction > 0 and coldstart_ef > 0:
1216
+ if (cstv_id, timevar_id) not in cstvs_cache:
1217
+ cstv_index = pd.date_range(
1218
+ start=begin - datetime.timedelta(hours=1),
1219
+ end=end - datetime.timedelta(hours=1),
1220
+ freq="h",
1221
+ )
1222
+ cstvs_cache[(cstv_id, timevar_id)] = timevar_to_series(
1223
+ cstv_index,
1224
+ self.coldstart_timevars[cstv_id or "default"],
1225
+ self.flow_timevars[timevar_id or "default"],
1226
+ timezone=self.timezone,
1227
+ )
1228
+ coldstart_timevar_scalings = cstvs_cache[
1229
+ (cstv_id, timevar_id)
1230
+ ]
1231
+
1232
+ # dataframe with intensity scalings for each hour
1233
+ flow_timevar_scalings = self.flow_timevar_scalings[
1234
+ timevar_id or "default"
1235
+ ]
1236
+ cong_prof_id = rec[col_map.congestion_profile_id or "default"]
1237
+
1238
+ # dataframe with traffic conditions (values 1-4)
1239
+ traffic_conditions = self.traffic_conditions[
1240
+ cong_prof_id or "default"
1241
+ ]
1242
+
1243
+ # calculate emission timeseries
1244
+ # exhaust emissions + coldstart emissions
1245
+ flow = rec[col_map.veh_m_per_sec]
1246
+ scaled_veh_flow = flow_timevar_scalings.to_numpy() * flow
1247
+
1248
+ freeflow_ef = rec[col_map.freeflow_ef]
1249
+ heavy_ef = rec[col_map.heavy_ef]
1250
+ saturated_ef = rec[col_map.saturated_ef]
1251
+ stopngo_ef = rec[col_map.stopngo_ef]
1252
+
1253
+ emis_fac_key = (
1254
+ cong_prof_id,
1255
+ freeflow_ef,
1256
+ heavy_ef,
1257
+ saturated_ef,
1258
+ stopngo_ef,
1259
+ )
1260
+ if emis_fac_key not in emis_fac_cache:
1261
+ # array for emission factors for current vehicle
1262
+ # one ef for each traffic condition
1263
+ emis_fac = emis_fac_freeflow * freeflow_ef
1264
+ emis_fac = np.where(
1265
+ traffic_conditions == 2, heavy_ef, emis_fac
1266
+ )
1267
+ emis_fac = np.where(
1268
+ traffic_conditions == 3, saturated_ef, emis_fac
1269
+ )
1270
+ emis_fac = np.where(
1271
+ traffic_conditions == 4, stopngo_ef, emis_fac
1272
+ )
1273
+ emis_fac_cache[emis_fac_key] = emis_fac
1274
+ else:
1275
+ emis_fac = emis_fac_cache[emis_fac_key]
1276
+
1277
+ if coldstart_ef > 0 and coldstart_fraction > 0:
1278
+ coldstart_scaled_veh_flow = (
1279
+ coldstart_timevar_scalings.to_numpy()
1280
+ * coldstart_fraction
1281
+ * coldstart_ef
1282
+ )
1283
+
1284
+ emis_ts = (
1285
+ scaled_veh_flow * emis_fac
1286
+ + flow * coldstart_scaled_veh_flow
1287
+ )
1288
+ else:
1289
+ emis_ts = scaled_veh_flow * emis_fac
1290
+
1291
+ else:
1292
+ # dataframe with intensity scalings for each hour
1293
+ timevar_scalings = self.source_timevar_scalings[
1294
+ timevar_id or "default"
1295
+ ]
1296
+
1297
+ # calculate emission timeseries scaled by timevar
1298
+ emis_ts = timevar_scalings.to_numpy() * emis
1299
+
1300
+ if sourcetype != GRID:
1301
+ # get cell indices and weights for source
1302
+ index_array, source_weights = weights[source_id]
1303
+ else:
1304
+ raster_name = rec[col_map.raster]
1305
+ try:
1306
+ index_array, source_weights = weights[
1307
+ (source_id, raster_name)
1308
+ ]
1309
+ except KeyError:
1310
+ # grid has no emissions within modelling area
1311
+ continue
1312
+ # add time dimension to index array
1313
+ try:
1314
+ ncells = index_array[0].size
1315
+ except AttributeError:
1316
+ ncells = len(index_array[0])
1317
+ index_array = (
1318
+ np.tile(np.arange(nr_timesteps), ncells),
1319
+ np.repeat(index_array[0], nr_timesteps),
1320
+ np.repeat(index_array[1], nr_timesteps),
1321
+ )
1322
+
1323
+ # organize 1D-array of emissions as the index array
1324
+ emis_array = (source_weights * emis_ts[:, np.newaxis]).T.reshape(-1)
1325
+
1326
+ # add emissions to chunk at each index
1327
+ chunk[index_array] += emis_array
1328
+ return chunk
1329
+
1330
+ def _calc_chunking(
1331
+ self, time_chunksize=None, spatial_chunksize=None, chunk_cache=1e7
1332
+ ):
1333
+ """Return chunksizes for variable dimensions."""
1334
+
1335
+ # if there is no time-dimension, chunking is not needed
1336
+ # for grids smaller than 100 x 100, chunksize covers the whole grid
1337
+ if len(self.timevars) == 0:
1338
+ return None
1339
+
1340
+ # ensure chunk size is not larger than grid
1341
+ chunk_nx = min(self.nx, spatial_chunksize or 5)
1342
+ chunk_ny = min(self.ny, spatial_chunksize or 5)
1343
+
1344
+ if time_chunksize is None:
1345
+ # assumes single precision (4 bytes per value)
1346
+ # max estimate based on full grid rather than limiting to spatial chunks,
1347
+ # (more practical if full grid fits in memory...)
1348
+
1349
+ field_size_bytes = self.nx * self.ny * 4
1350
+ max_time_chunksize = int(chunk_cache / field_size_bytes)
1351
+
1352
+ if self.time_step is not None:
1353
+ time_chunksize = int(
1354
+ datetime.timedelta(hours=366 * 24)
1355
+ / time_step_as_delta(self.time_step)
1356
+ )
1357
+ else:
1358
+ time_chunksize = 366
1359
+
1360
+ time_chunksize = min(time_chunksize, max_time_chunksize)
1361
+
1362
+ return (time_chunksize, chunk_ny, chunk_nx)
1363
+
1364
+ def create_gridmapping_variable(self, dset: nc.Dataset, crs: rio.CRS):
1365
+ name = f"EPSG_{crs.to_epsg()}"
1366
+ grid_mapping = dset.createVariable(name, "i")
1367
+ epsg = crs.to_epsg()
1368
+ if epsg is None:
1369
+ epsg = crs.to_epsg(confidence_threshold=30)
1370
+ self.log.info(
1371
+ f"No exact match for EPSG and spatial ref, using best guess: {epsg}"
1372
+ )
1373
+ grid_mapping.srid = epsg
1374
+ grid_mapping.crs_wkt = crs.to_wkt() # for CF Conventions 1.7
1375
+ return grid_mapping
1376
+
1377
+
1378
+ def create_variable(
1379
+ dset: nc.Dataset,
1380
+ grid_mapping: nc.Variable,
1381
+ name: str,
1382
+ unit: Optional[str] = None,
1383
+ quantity: Optional[str] = None,
1384
+ instance: Optional[str] = None,
1385
+ cell_methods: Optional[str] = None,
1386
+ parameter: Optional[str] = None,
1387
+ height: Optional[float] = None,
1388
+ time: bool = True,
1389
+ chunksizes: Optional[tuple[int, ...]] = None,
1390
+ ):
1391
+ """create 2D variable."""
1392
+ if time:
1393
+ var = dset.createVariable(
1394
+ name,
1395
+ "f4",
1396
+ (TIME_NAME, Y_NAME, X_NAME),
1397
+ fill_value=np.nan,
1398
+ zlib=True,
1399
+ chunksizes=chunksizes,
1400
+ )
1401
+ else:
1402
+ var = dset.createVariable(
1403
+ name, "f4", (Y_NAME, X_NAME), fill_value=np.nan, zlib=True
1404
+ )
1405
+
1406
+ var.units = unit or "unknown"
1407
+ var.grid_mapping = grid_mapping.name
1408
+ if parameter is not None:
1409
+ var.parameter = parameter
1410
+ if quantity is not None:
1411
+ var.quantity = quantity
1412
+ if instance is not None:
1413
+ var.instance = instance
1414
+ if cell_methods is not None:
1415
+ var.cell_methods = cell_methods
1416
+ if height is not None:
1417
+ var.height = height
1418
+ return var
1419
+
1420
+
1421
+ GENERAL_ATTRIBUTES = [("Created_using_cetk", __version__)]
1422
+
1423
+
1424
+ def write_general_attrs(dset: nc.Dataset):
1425
+ """add some global attributes."""
1426
+ for name, attr in GENERAL_ATTRIBUTES:
1427
+ if not hasattr(dset, name):
1428
+ setattr(dset, name, attr)
1429
+
1430
+ if not hasattr(dset, "history"):
1431
+ timestamp = datetime.datetime.now(tz=datetime.timezone.utc).strftime(
1432
+ "%Y%m%d %H:%M"
1433
+ )
1434
+ dset.history = f"{timestamp} created dataset"
1435
+
1436
+
1437
+ # name of time-dimension and time-variable
1438
+ TIME_NAME = "time"
1439
+ X_NAME = "x"
1440
+ Y_NAME = "y"
1441
+ # name of time-bounds dimension and time-bounds variable
1442
+ TIME_BOUNDS_NAME = "time_bounds"
1443
+ # name of second dimension for bounds variables
1444
+ BOUNDS_DIMENSION = "bounds_dim"
1445
+ # netCDF reference time
1446
+ TIME_ZERO = datetime.datetime(
1447
+ year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=datetime.timezone.utc
1448
+ )
1449
+
1450
+
1451
+ def create_time_variable(dset: nc.Dataset):
1452
+ """create time variable."""
1453
+
1454
+ times = dset.createDimension(TIME_NAME, None)
1455
+ dset.createDimension(BOUNDS_DIMENSION, 2)
1456
+ times = dset.createVariable(TIME_NAME, "f8", (TIME_NAME,))
1457
+ times.calendar = "gregorian"
1458
+ times.units = f"hours since {TIME_ZERO:%Y-%m-%d %H:%M:%S}"
1459
+ times.bounds = TIME_BOUNDS_NAME
1460
+ times.long_name = "time"
1461
+ times.axis = "T"
1462
+ time_bounds = dset.createVariable(
1463
+ TIME_BOUNDS_NAME, "f8", (TIME_NAME, BOUNDS_DIMENSION)
1464
+ )
1465
+ return times, time_bounds
1466
+
1467
+
1468
+ def create_xy_variables(
1469
+ dset: nc.Dataset, extent: tuple, crs: rio.CRS, nx: int, ny: int
1470
+ ):
1471
+ """Create a netCDF dataset with dimensions and metadata for 2D fields."""
1472
+
1473
+ x1, y1, x2, y2 = extent
1474
+ dx = (x2 - x1) / nx
1475
+ dy = (y2 - y1) / ny
1476
+
1477
+ dset.createDimension(X_NAME, nx)
1478
+ # create projected coordinate variables
1479
+ x = dset.createVariable(X_NAME, "f4", (X_NAME,))
1480
+ x.standard_name = "projection_x_coordinate"
1481
+ x.units = "m"
1482
+ x.axis = "X"
1483
+ x.description = "center of cell"
1484
+ x[:] = np.linspace(x1 + 0.5 * dx, x2 - 0.5 * dx, nx)
1485
+
1486
+ dset.createDimension(Y_NAME, ny)
1487
+ y = dset.createVariable(Y_NAME, "f4", (Y_NAME,))
1488
+ y.standard_name = "projection_y_coordinate"
1489
+ y.units = "m"
1490
+ y.axis = "Y"
1491
+ y.description = "center of cell"
1492
+ y[:] = np.linspace(y1 + 0.5 * dy, y2 - 0.5 * dy, ny)
1493
+ return x, y
1494
+
1495
+
1496
+ def make_naive_in_utc(time: datetime.datetime):
1497
+ """convert time to utc and make naive."""
1498
+ if time.tzinfo is not None:
1499
+ time_utc = time.astimezone(datetime.timezone.utc)
1500
+ time_utc.replace(tzinfo=None)
1501
+ else:
1502
+ time_utc = time
1503
+ return time_utc
1504
+
1505
+
1506
+ def write_time(
1507
+ time_var: nc.Variable, time_bounds_var: nc.Variable, time_utc: datetime.datetime
1508
+ ):
1509
+ """write timestamp to file and return time-index"""
1510
+
1511
+ left_bound = time_utc - datetime.timedelta(hours=1)
1512
+ # get index of time in netCDF
1513
+ if len(time_var) == 0:
1514
+ time_index = 0
1515
+ else:
1516
+ time_index = nc.date2index(time_utc, time_var, select="before") + 1
1517
+
1518
+ # write time to netCDF (in numerical format)
1519
+ nc_time = nc.date2num(time_utc, time_var.units)
1520
+ nc_left_time_bound = nc.date2num(left_bound, time_var.units)
1521
+ nc_right_time_bound = nc_time
1522
+
1523
+ time_bounds_var[time_index, :] = [nc_left_time_bound, nc_right_time_bound]
1524
+ time_var[time_index] = nc_time
1525
+ return time_index
1526
+
1527
+
1528
+ def get_nodes_from_wkt(wkt):
1529
+ """Extract list of nodes (x, y) from WKT.
1530
+
1531
+ geom_type: 'POINT', 'LINESTRING' or 'POLYGON'
1532
+
1533
+ """
1534
+ geom = GEOSGeometry(wkt)
1535
+ if geom.geom_type == "Point":
1536
+ return (geom.coords,)
1537
+ if geom.geom_type == "LineString":
1538
+ return geom.coords
1539
+ if geom.geom_type == "Polygon":
1540
+ return geom.coords[0]
1541
+
1542
+ raise NotImplementedError(
1543
+ f'Geometry type specified in WKT not implemented: "{wkt}"'
1544
+ )