pyelq 1.1.3__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyelq/__init__.py +1 -0
- pyelq/component/__init__.py +1 -0
- pyelq/component/background.py +19 -13
- pyelq/component/component.py +2 -1
- pyelq/component/error_model.py +2 -1
- pyelq/component/offset.py +2 -1
- pyelq/component/source_model.py +78 -29
- pyelq/coordinate_system.py +1 -0
- pyelq/data_access/__init__.py +1 -0
- pyelq/data_access/data_access.py +1 -1
- pyelq/dispersion_model/__init__.py +4 -3
- pyelq/dispersion_model/dispersion_model.py +202 -0
- pyelq/dispersion_model/finite_volume.py +1084 -0
- pyelq/dispersion_model/gaussian_plume.py +8 -189
- pyelq/dispersion_model/site_layout.py +97 -0
- pyelq/dlm.py +11 -15
- pyelq/gas_species.py +1 -0
- pyelq/meteorology/__init__.py +6 -0
- pyelq/{meteorology.py → meteorology/meteorology.py} +388 -387
- pyelq/meteorology/meteorology_windfield.py +180 -0
- pyelq/model.py +2 -1
- pyelq/plotting/__init__.py +1 -0
- pyelq/plotting/plot.py +16 -25
- pyelq/preprocessing.py +98 -38
- pyelq/sensor/__init__.py +1 -0
- pyelq/sensor/sensor.py +70 -5
- pyelq/source_map.py +1 -0
- pyelq/support_functions/__init__.py +1 -0
- pyelq/support_functions/post_processing.py +1 -0
- pyelq/support_functions/spatio_temporal_interpolation.py +1 -0
- {pyelq-1.1.3.dist-info → pyelq-1.2.0.dist-info}/METADATA +45 -44
- pyelq-1.2.0.dist-info/RECORD +37 -0
- {pyelq-1.1.3.dist-info → pyelq-1.2.0.dist-info}/WHEEL +1 -1
- pyelq-1.1.3.dist-info/RECORD +0 -32
- {pyelq-1.1.3.dist-info → pyelq-1.2.0.dist-info/licenses}/LICENSE.md +0 -0
- {pyelq-1.1.3.dist-info → pyelq-1.2.0.dist-info/licenses}/LICENSES/Apache-2.0.txt +0 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Shell Global Solutions International B.V. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
# -*- coding: utf-8 -*-
|
|
6
|
+
"""Meteorology windfield module.
|
|
7
|
+
|
|
8
|
+
Version of the meteorology class that deals with spatial wind fields and can calculate the wind field around
|
|
9
|
+
cylindrical obstacles.
|
|
10
|
+
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Optional, Tuple, Union
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from pyelq.coordinate_system import ENU
|
|
19
|
+
from pyelq.dispersion_model.site_layout import SiteLayout
|
|
20
|
+
from pyelq.meteorology.meteorology import Meteorology
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class MeteorologyWindfield(Meteorology):
|
|
25
|
+
"""Represents a spatially resolved wind field based on meteorological measurements and the presence of obstacles.
|
|
26
|
+
|
|
27
|
+
This class extends the base `Meteorology` class by providing methods to compute the local wind vector (u and v
|
|
28
|
+
components) at every grid point, factoring in obstacle perturbations using an analytical method. It accounts for
|
|
29
|
+
spatial rotation to align with the instantaneous wind direction at each time step.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
static_wind_field (Meteorology): The static wind field used for calculations.
|
|
33
|
+
site_layout (SiteLayout): The layout of the site, including cylinder coordinates and radii.
|
|
34
|
+
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
static_wind_field: Meteorology
|
|
38
|
+
site_layout: Optional[Union[SiteLayout, None]] = None
|
|
39
|
+
|
|
40
|
+
def calculate_spatial_wind_field(self, grid_coordinates: ENU, time_index: int = None):
|
|
41
|
+
"""Calculates the spatial wind field over a grid considering obstacles.
|
|
42
|
+
|
|
43
|
+
Computes the full spatial wind field from a time series stored in self.static_wind_field at grid locations
|
|
44
|
+
provided in grid_coordinates considering distortion effects due to flow around cylindrical obstacles.
|
|
45
|
+
|
|
46
|
+
The method:
|
|
47
|
+
- Rotates grid coordinates into the wind-aligned frame based on mathematical wind direction.
|
|
48
|
+
- Calculates the distorted wind field due to the presence of cylindrical obstacles.
|
|
49
|
+
- Rotates the resulting local wind field back into the original frame.
|
|
50
|
+
- Updates the object's `u_component` and `v_component` accordingly.
|
|
51
|
+
- If w_component is present in the static wind field, it is broadcasted to match the grid points.
|
|
52
|
+
- If no site layout is provided, the wind field remains undisturbed and is simply broadcasted across the grid.
|
|
53
|
+
|
|
54
|
+
Output: The method updates the following properties in place:
|
|
55
|
+
- u_component np.ndarray: (n_grid x n_time) The x-component of the wind field at the grid points.
|
|
56
|
+
- v_component np.ndarray: (n_grid x n_time) The y-component of the wind field at the grid points.
|
|
57
|
+
- w_component np.ndarray: (n_grid x n_time) The z-component of the wind field at the grid points.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
grid_coordinates (ENU): The coordinates of the grid points.
|
|
61
|
+
time_index (int): The time index for the meteorological data.
|
|
62
|
+
|
|
63
|
+
"""
|
|
64
|
+
if time_index is None:
|
|
65
|
+
time_index = np.ones_like(self.static_wind_field.u_component).astype(bool)
|
|
66
|
+
|
|
67
|
+
u = self.static_wind_field.u_component.reshape(-1, 1)[time_index]
|
|
68
|
+
v = self.static_wind_field.v_component.reshape(-1, 1)[time_index]
|
|
69
|
+
if self.static_wind_field.w_component is not None:
|
|
70
|
+
self.w_component = np.broadcast_to(
|
|
71
|
+
self.static_wind_field.w_component[time_index].T,
|
|
72
|
+
(grid_coordinates.nof_observations, u.shape[0]),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
if self.site_layout is None:
|
|
76
|
+
self.u_component = np.broadcast_to(u.T, (grid_coordinates.nof_observations, u.shape[0]))
|
|
77
|
+
self.v_component = np.broadcast_to(v.T, (grid_coordinates.nof_observations, v.shape[0]))
|
|
78
|
+
return
|
|
79
|
+
mathematical_wind_direction = np.arctan2(v, u).flatten()
|
|
80
|
+
rotation_matrix, rotated_grid, rotated_cylinders = self._rotate_coordinates(
|
|
81
|
+
grid_coordinates, mathematical_wind_direction
|
|
82
|
+
)
|
|
83
|
+
u_rot, v_rot = self._calculate_wind_field_cardinal(
|
|
84
|
+
u=u,
|
|
85
|
+
v=v,
|
|
86
|
+
grid_coordinates=grid_coordinates,
|
|
87
|
+
rotated_grid=rotated_grid,
|
|
88
|
+
rotated_cylinders=rotated_cylinders,
|
|
89
|
+
)
|
|
90
|
+
u_stacked = np.stack((u_rot, v_rot), axis=2)
|
|
91
|
+
inverse_rot = np.transpose(rotation_matrix, axes=(1, 0, 2))
|
|
92
|
+
rotated_wind = np.einsum("ijt,ntj-> nti", inverse_rot, u_stacked)
|
|
93
|
+
self.u_component = rotated_wind[:, :, 0]
|
|
94
|
+
self.v_component = rotated_wind[:, :, 1]
|
|
95
|
+
|
|
96
|
+
def _rotate_coordinates(
|
|
97
|
+
self, grid_coordinates: ENU, wind_direction: np.ndarray
|
|
98
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
99
|
+
"""Rotates the x, y coordinates based on the wind direction.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
grid_coordinates (ENU): The coordinates to be rotated.
|
|
103
|
+
wind_direction (np.array): The wind direction in radians.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
rotation_matrix (np.ndarray): rotation matrix used for inverse rotation
|
|
107
|
+
rotated_grid (np.ndarray): grid_coordinates in rotated coordinate system
|
|
108
|
+
rotated_cylinders (np.ndarray): cylinder locations in rotated coordinate system
|
|
109
|
+
|
|
110
|
+
"""
|
|
111
|
+
rotation_matrix = np.array(
|
|
112
|
+
[
|
|
113
|
+
[np.cos(wind_direction), np.sin(wind_direction)],
|
|
114
|
+
[-np.sin(wind_direction), np.cos(wind_direction)],
|
|
115
|
+
]
|
|
116
|
+
)
|
|
117
|
+
rotated_grid = np.einsum("ijt,nj->nit", rotation_matrix, grid_coordinates.to_array(dim=2))
|
|
118
|
+
rotated_cylinders = np.einsum(
|
|
119
|
+
"ijt,nj->nit", rotation_matrix, self.site_layout.cylinder_coordinates.to_array(dim=2)
|
|
120
|
+
)
|
|
121
|
+
return rotation_matrix, rotated_grid, rotated_cylinders
|
|
122
|
+
|
|
123
|
+
def _calculate_wind_field_cardinal(
|
|
124
|
+
self,
|
|
125
|
+
u: np.ndarray,
|
|
126
|
+
v: np.ndarray,
|
|
127
|
+
grid_coordinates: ENU,
|
|
128
|
+
rotated_grid: np.ndarray,
|
|
129
|
+
rotated_cylinders: np.ndarray,
|
|
130
|
+
) -> Tuple[np.ndarray, np.ndarray]:
|
|
131
|
+
"""Calculates the distorted wind field components (u, v) in the wind-aligned (cardinal) frame.
|
|
132
|
+
|
|
133
|
+
The method:
|
|
134
|
+
- Determines whether each grid point is influenced by nearby cylinders based on distance and cylinder radius.
|
|
135
|
+
- If no obstacles are relevant at the evaluation height, the wind field remains undisturbed.
|
|
136
|
+
- If obstacles are present, modifies the wind field using an analytical perturbation formula based on
|
|
137
|
+
potential flow theory.
|
|
138
|
+
- Wind inside obstacles is set to zero.
|
|
139
|
+
|
|
140
|
+
If no height information is provided (e.g. in the 2-dimensional solver case), the function assumes that
|
|
141
|
+
all cylinders and input points are at the same height, and applies the mask accordingly.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
u (np.ndarray n_time x 1): The x-component of the wind vector.
|
|
145
|
+
v (np.ndarray n_time x 1): The y-component of the wind vector.
|
|
146
|
+
grid_coordinates (ENU): location object containing information about the finite volume solve grid points.
|
|
147
|
+
rotated_grid (np.ndarray n_grid x 2 x n_time): The grid coordinates where the wind field is to be calculated
|
|
148
|
+
in the wind-aligned frame.
|
|
149
|
+
rotated_cylinders (np.ndarray n_cylinders x 2 x n_time): The coordinates of the cylinders in the
|
|
150
|
+
wind-aligned frame.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
u_rot (np.ndarray): The x-component of the wind field at the grid points.
|
|
154
|
+
v_rot (np.ndarray): The y-component of the wind field at the grid points.
|
|
155
|
+
|
|
156
|
+
"""
|
|
157
|
+
diff = rotated_grid[:, np.newaxis, :, :] - rotated_cylinders[np.newaxis, :, :, :]
|
|
158
|
+
radial_distance = np.linalg.norm(diff, axis=2)
|
|
159
|
+
x_diff = diff[:, :, 0, :]
|
|
160
|
+
y_diff = diff[:, :, 1, :]
|
|
161
|
+
radius_squared = self.site_layout.cylinder_radius.T**2
|
|
162
|
+
radial_distance_sq = radial_distance**2
|
|
163
|
+
radial_distance_quad = radial_distance_sq**2
|
|
164
|
+
radial_distance_quad[radial_distance_quad == 0] = np.nan
|
|
165
|
+
radius_sq_over_r4 = radius_squared[:, :, np.newaxis] / radial_distance_quad
|
|
166
|
+
|
|
167
|
+
if grid_coordinates.up is None:
|
|
168
|
+
sum_term_x = np.einsum("nct, nct->nt", radius_sq_over_r4, (y_diff**2 - x_diff**2))
|
|
169
|
+
sum_term_y = np.einsum("nct, nct->nt", radius_sq_over_r4, (y_diff * x_diff))
|
|
170
|
+
else:
|
|
171
|
+
height_mask = grid_coordinates.up <= self.site_layout.cylinder_coordinates.up.T
|
|
172
|
+
height_mask = height_mask.reshape(grid_coordinates.nof_observations, self.site_layout.nof_cylinders)
|
|
173
|
+
sum_term_x = np.einsum("nc, nct, nct->nt", height_mask, radius_sq_over_r4, (y_diff**2 - x_diff**2))
|
|
174
|
+
sum_term_y = np.einsum("nc, nct, nct->nt", height_mask, radius_sq_over_r4, (y_diff * x_diff))
|
|
175
|
+
wind_speed = np.sqrt(u**2 + v**2).T
|
|
176
|
+
u_rot = wind_speed * (1 + sum_term_x)
|
|
177
|
+
v_rot = -2 * wind_speed * sum_term_y
|
|
178
|
+
u_rot[self.site_layout.id_obstacles.flatten(), :] = 0
|
|
179
|
+
v_rot[self.site_layout.id_obstacles.flatten(), :] = 0
|
|
180
|
+
return u_rot, v_rot
|
pyelq/model.py
CHANGED
|
@@ -9,6 +9,7 @@ This module provides a class definition for the main functionalities of the code
|
|
|
9
9
|
openMCMC repo and defining some plotting wrappers.
|
|
10
10
|
|
|
11
11
|
"""
|
|
12
|
+
|
|
12
13
|
import re
|
|
13
14
|
import warnings
|
|
14
15
|
from dataclasses import dataclass, field
|
|
@@ -26,7 +27,7 @@ from pyelq.component.offset import PerSensor
|
|
|
26
27
|
from pyelq.component.source_model import Normal, SourceModel
|
|
27
28
|
from pyelq.coordinate_system import ENU
|
|
28
29
|
from pyelq.gas_species import GasSpecies
|
|
29
|
-
from pyelq.meteorology import Meteorology, MeteorologyGroup
|
|
30
|
+
from pyelq.meteorology.meteorology import Meteorology, MeteorologyGroup
|
|
30
31
|
from pyelq.plotting.plot import Plot
|
|
31
32
|
from pyelq.sensor.sensor import SensorGroup
|
|
32
33
|
|
pyelq/plotting/__init__.py
CHANGED
pyelq/plotting/plot.py
CHANGED
|
@@ -9,6 +9,7 @@ Large module containing all the plotting code used to create various plots. Cont
|
|
|
9
9
|
definition.
|
|
10
10
|
|
|
11
11
|
"""
|
|
12
|
+
|
|
12
13
|
import re
|
|
13
14
|
import warnings
|
|
14
15
|
from copy import deepcopy
|
|
@@ -978,6 +979,7 @@ class Plot:
|
|
|
978
979
|
normalized_count_limit: float = 0.005,
|
|
979
980
|
burn_in: int = 0,
|
|
980
981
|
show_summary_results: bool = True,
|
|
982
|
+
show_fixed_source_locations: bool = True,
|
|
981
983
|
):
|
|
982
984
|
"""Function to create a map with the quantification results of the model object.
|
|
983
985
|
|
|
@@ -996,6 +998,8 @@ class Plot:
|
|
|
996
998
|
burn_in (int, optional): Number of burn-in iterations to discard before calculating the statistics.
|
|
997
999
|
Defaults to 0.
|
|
998
1000
|
show_summary_results (bool, optional): Flag to show the summary results on the map. Defaults to True.
|
|
1001
|
+
show_fixed_source_locations (bool, optional): Flag to show the fixed sources location when present in one
|
|
1002
|
+
of the sourcemaps. Defaults to True.
|
|
999
1003
|
|
|
1000
1004
|
"""
|
|
1001
1005
|
if source_model_to_plot_key is None:
|
|
@@ -1109,33 +1113,20 @@ class Plot:
|
|
|
1109
1113
|
sensor_object.plot_sensor_location(self.figure_dict["iqr_map"])
|
|
1110
1114
|
self.figure_dict["iqr_map"].update_traces(showlegend=False)
|
|
1111
1115
|
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
latitude=np.nanmean(source_location_fixed_lla.latitude, axis=1),
|
|
1121
|
-
longitude=np.nanmean(source_location_fixed_lla.longitude, axis=1),
|
|
1122
|
-
altitude=np.nanmean(source_location_fixed_lla.altitude, axis=1),
|
|
1123
|
-
)
|
|
1124
|
-
|
|
1125
|
-
for lat_fixed, lon_fixed, label_fixed in zip(
|
|
1126
|
-
source_location_fixed_average.latitude,
|
|
1127
|
-
source_location_fixed_average.longitude,
|
|
1128
|
-
source_model_fixed.individual_source_labels,
|
|
1129
|
-
):
|
|
1130
|
-
color_idx = source_model_fixed.individual_source_labels.index(label_fixed)
|
|
1131
|
-
marker_dict["color"] = colormap_fixed[color_idx % len(colormap_fixed)]
|
|
1132
|
-
|
|
1116
|
+
if show_fixed_source_locations:
|
|
1117
|
+
for key, _ in model_object.components.items():
|
|
1118
|
+
if bool(re.search("fixed", key)):
|
|
1119
|
+
source_model_fixed = model_object.components[key]
|
|
1120
|
+
source_locations_fixed = source_model_fixed.all_source_locations
|
|
1121
|
+
source_location_fixed_lla = source_locations_fixed.to_lla()
|
|
1122
|
+
sources_lat = source_location_fixed_lla.latitude[:, 0]
|
|
1123
|
+
sources_lon = source_location_fixed_lla.longitude[:, 0]
|
|
1133
1124
|
fixed_source_location_trace = go.Scattermap(
|
|
1134
1125
|
mode="markers",
|
|
1135
|
-
lon=
|
|
1136
|
-
lat=
|
|
1137
|
-
name=
|
|
1138
|
-
marker=
|
|
1126
|
+
lon=sources_lon,
|
|
1127
|
+
lat=sources_lat,
|
|
1128
|
+
name=f"Fixed source locations, {key}",
|
|
1129
|
+
marker={"size": 10, "opacity": 0.8},
|
|
1139
1130
|
)
|
|
1140
1131
|
self.figure_dict["count_map"].add_trace(fixed_source_location_trace)
|
|
1141
1132
|
self.figure_dict["median_map"].add_trace(fixed_source_location_trace)
|
pyelq/preprocessing.py
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024 Shell Global Solutions International B.V. All Rights Reserved.
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
|
|
5
4
|
"""Class for performing preprocessing on the loaded data."""
|
|
6
5
|
|
|
7
6
|
from copy import deepcopy
|
|
8
|
-
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import dataclass, field
|
|
9
8
|
from typing import Union
|
|
10
9
|
|
|
11
10
|
import numpy as np
|
|
12
11
|
import pandas as pd
|
|
13
12
|
|
|
14
|
-
from pyelq.meteorology import Meteorology, MeteorologyGroup
|
|
13
|
+
from pyelq.meteorology.meteorology import Meteorology, MeteorologyGroup
|
|
15
14
|
from pyelq.sensor.sensor import Sensor, SensorGroup
|
|
16
15
|
from pyelq.support_functions.spatio_temporal_interpolation import temporal_resampling
|
|
17
16
|
|
|
@@ -21,16 +20,17 @@ class Preprocessor:
|
|
|
21
20
|
"""Class which implements generic functionality for pre-processing of sensor and meteorology information.
|
|
22
21
|
|
|
23
22
|
Attributes:
|
|
24
|
-
time_bin_edges (pd.arrays.DatetimeArray): edges of the time bins to be used for
|
|
23
|
+
time_bin_edges (Union[pd.arrays.DatetimeArray, None]): edges of the time bins to be used for
|
|
24
|
+
smoothing/interpolation. If None, no smoothing/interpolation is performed.
|
|
25
25
|
sensor_object (SensorGroup): sensor group object containing raw data.
|
|
26
26
|
met_object (Meteorology): met object containing raw data.
|
|
27
27
|
aggregate_function (str): function to be used for aggregation of data. Defaults to mean.
|
|
28
28
|
sensor_fields (list): standard list of sensor attributes that we wish to regularize and/or filter.
|
|
29
29
|
met_fields (list): standard list of meteorology attributes that we wish to regularize/filter.
|
|
30
|
-
|
|
30
|
+
is_regularized (bool): flag indicating whether the met and sensor data has been regularized.
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
|
-
time_bin_edges: pd.arrays.DatetimeArray
|
|
33
|
+
time_bin_edges: Union[pd.arrays.DatetimeArray, None]
|
|
34
34
|
sensor_object: SensorGroup
|
|
35
35
|
met_object: Union[Meteorology, MeteorologyGroup]
|
|
36
36
|
aggregate_function: str = "mean"
|
|
@@ -47,27 +47,36 @@ class Preprocessor:
|
|
|
47
47
|
"wind_turbulence_horizontal",
|
|
48
48
|
"wind_turbulence_vertical",
|
|
49
49
|
]
|
|
50
|
+
is_regularized: bool = field(init=False, default=False)
|
|
50
51
|
|
|
51
52
|
def __post_init__(self) -> None:
|
|
52
53
|
"""Initialise the class.
|
|
53
54
|
|
|
54
55
|
Attaching the sensor and meteorology objects as attributes, and running initial regularization and NaN filtering
|
|
55
|
-
steps.
|
|
56
|
+
steps. If time_bin_edges is provided, the data will be smoothed/interpolated onto the specified time grid and
|
|
57
|
+
NaN values will be filtered out. If time_bin_edges is None, the data will be filtered for NaN values only.
|
|
56
58
|
|
|
57
|
-
Before running the regularization & NaN filtering, the function
|
|
58
|
-
present as fields on met_object. The post-smoothing wind speed and
|
|
59
|
-
smoothed u and v components, to eliminate the need to take means of
|
|
59
|
+
Before running the regularization (for the case where time_bin_edges is provided) & NaN filtering, the function
|
|
60
|
+
ensures that u_component and v_component are present as fields on met_object. The post-smoothing wind speed and
|
|
61
|
+
direction are then calculated from the smoothed u and v components, to eliminate the need to take means of
|
|
62
|
+
directions when binning.
|
|
60
63
|
|
|
61
|
-
|
|
62
|
-
device, identical time stamps, and be free of NaNs.
|
|
64
|
+
If time_bin_edges is provided, the sensor and meteorology group objects attached to the class will have
|
|
65
|
+
identical numbers of data points per device, identical time stamps, and be free of NaNs. If time_bin_edges is
|
|
66
|
+
None, the sensor and meteorology group objects will not have identical time stamps, but will be free of NaNs.
|
|
67
|
+
The time stamps for the sensor objects and meteorology objects will be the original time stamps.
|
|
63
68
|
|
|
64
69
|
"""
|
|
65
70
|
self.met_object.calculate_uv_from_wind_speed_direction()
|
|
71
|
+
if self.time_bin_edges is not None:
|
|
72
|
+
self.regularize_data()
|
|
66
73
|
|
|
67
|
-
self.regularize_data()
|
|
68
74
|
self.met_object.calculate_wind_direction_from_uv()
|
|
69
75
|
self.met_object.calculate_wind_speed_from_uv()
|
|
70
|
-
self.
|
|
76
|
+
if self.is_regularized:
|
|
77
|
+
self.filter_nans()
|
|
78
|
+
else:
|
|
79
|
+
self.filter_nans_no_regularize()
|
|
71
80
|
|
|
72
81
|
def regularize_data(self) -> None:
|
|
73
82
|
"""Smoothing or interpolation of data onto a common set of time points.
|
|
@@ -88,6 +97,7 @@ class Preprocessor:
|
|
|
88
97
|
will have identical time stamps, but may still contain NaNs.
|
|
89
98
|
|
|
90
99
|
"""
|
|
100
|
+
self.is_regularized = True
|
|
91
101
|
sensor_out = deepcopy(self.sensor_object)
|
|
92
102
|
for sns_new, sns_old in zip(sensor_out.values(), self.sensor_object.values()):
|
|
93
103
|
for field in self.sensor_fields:
|
|
@@ -126,23 +136,66 @@ class Preprocessor:
|
|
|
126
136
|
for sns_key, met_key in zip(self.sensor_object, self.met_object):
|
|
127
137
|
sns_in = self.sensor_object[sns_key]
|
|
128
138
|
met_in = self.met_object[met_key]
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
for field in self.met_fields:
|
|
134
|
-
if (field != "time") and (getattr(met_in, field) is not None):
|
|
135
|
-
filter_index = np.logical_and(filter_index, np.logical_not(np.isnan(getattr(met_in, field))))
|
|
139
|
+
|
|
140
|
+
filter_index_sensor = self.get_nan_filter_index(sns_in, self.sensor_fields)
|
|
141
|
+
filter_index_met = self.get_nan_filter_index(met_in, self.met_fields)
|
|
142
|
+
filter_index = np.logical_and(filter_index_sensor, filter_index_met)
|
|
136
143
|
|
|
137
144
|
self.sensor_object[sns_key] = self.filter_object_fields(sns_in, self.sensor_fields, filter_index)
|
|
138
145
|
self.met_object[met_key] = self.filter_object_fields(met_in, self.met_fields, filter_index)
|
|
139
146
|
|
|
147
|
+
def filter_nans_no_regularize(self) -> None:
|
|
148
|
+
"""Filter out data points where any of the specified sensor or meteorology fields has a NaN value.
|
|
149
|
+
|
|
150
|
+
Function first works through all sensor and meteorology fields and finds indices of all times where there is a
|
|
151
|
+
NaN value in any field. Then, it uses the resulting index to filter all fields.
|
|
152
|
+
The sensor_object and met_object are not assumed to have the same time stamps and they are treated separately.
|
|
153
|
+
|
|
154
|
+
The result of this function is that the sensor_object and met_object attributes of the class are updated, any
|
|
155
|
+
NaN values having been removed.
|
|
156
|
+
|
|
157
|
+
"""
|
|
158
|
+
for sns_key in self.sensor_object:
|
|
159
|
+
sns_in = self.sensor_object[sns_key]
|
|
160
|
+
filter_index = self.get_nan_filter_index(sns_in, self.sensor_fields)
|
|
161
|
+
self.sensor_object[sns_key] = self.filter_object_fields(sns_in, self.sensor_fields, filter_index)
|
|
162
|
+
|
|
163
|
+
if isinstance(self.met_object, Meteorology):
|
|
164
|
+
filter_index = self.get_nan_filter_index(self.met_object, self.met_fields)
|
|
165
|
+
self.met_object = self.filter_object_fields(self.met_object, self.met_fields, filter_index)
|
|
166
|
+
else:
|
|
167
|
+
raise TypeError("MeteorologyGroup not required in case with no regularization.")
|
|
168
|
+
|
|
169
|
+
@staticmethod
|
|
170
|
+
def get_nan_filter_index(obj: Union[Sensor, Meteorology], field_list: list) -> np.ndarray:
|
|
171
|
+
"""Get a index for a given object to be able to filter out on NaN values in listed fields.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
obj: Sensor or Meteorology object.
|
|
175
|
+
field_list (list): list of field names to be checked for NaN values.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
filter_index (np.ndarray): boolean array indicating which indices do not have NaN values in
|
|
179
|
+
any of the specified fields.
|
|
180
|
+
|
|
181
|
+
"""
|
|
182
|
+
filter_index = np.ones(obj.nof_observations, dtype=bool)
|
|
183
|
+
|
|
184
|
+
for field_name in field_list:
|
|
185
|
+
if (field_name != "time") and (getattr(obj, field_name) is not None):
|
|
186
|
+
filter_index = np.logical_and(filter_index, np.logical_not(np.isnan(getattr(obj, field_name))))
|
|
187
|
+
|
|
188
|
+
return filter_index
|
|
189
|
+
|
|
140
190
|
def filter_on_met(self, filter_variable: list, lower_limit: list = None, upper_limit: list = None) -> None:
|
|
141
191
|
"""Filter the supplied data on given properties of the meteorological data.
|
|
142
192
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
device.
|
|
193
|
+
If self.is_regularized, the filtering is done on both sensor_object and met_object and assumes that the
|
|
194
|
+
SensorGroup and MeteorologyGroup objects attached as attributes have corresponding values (one per sensor
|
|
195
|
+
device), and have attributes that have been pre-smoothed/interpolated onto a common time grid per device.
|
|
196
|
+
|
|
197
|
+
If the data is not regularized, the filtering is done on the met_object only. In this case a MeteorologyGroup
|
|
198
|
+
is not allowed.
|
|
146
199
|
|
|
147
200
|
The result of this function is that the sensor_object and met_object attributes are updated with the filtered
|
|
148
201
|
versions.
|
|
@@ -160,13 +213,20 @@ class Preprocessor:
|
|
|
160
213
|
if upper_limit is None:
|
|
161
214
|
upper_limit = [np.inf] * len(filter_variable)
|
|
162
215
|
|
|
163
|
-
|
|
164
|
-
for
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
216
|
+
if self.is_regularized:
|
|
217
|
+
for vrb, low, high in zip(filter_variable, lower_limit, upper_limit):
|
|
218
|
+
for sns_key, met_key in zip(self.sensor_object, self.met_object):
|
|
219
|
+
sns_in = self.sensor_object[sns_key]
|
|
220
|
+
met_in = self.met_object[met_key]
|
|
221
|
+
index_keep = np.logical_and(getattr(met_in, vrb) >= low, getattr(met_in, vrb) <= high)
|
|
222
|
+
self.sensor_object[sns_key] = self.filter_object_fields(sns_in, self.sensor_fields, index_keep)
|
|
223
|
+
self.met_object[met_key] = self.filter_object_fields(met_in, self.met_fields, index_keep)
|
|
224
|
+
else:
|
|
225
|
+
if isinstance(self.met_object, MeteorologyGroup):
|
|
226
|
+
raise TypeError("MeteorologyGroup not required in case with no regularization.")
|
|
227
|
+
for vrb, low, high in zip(filter_variable, lower_limit, upper_limit):
|
|
228
|
+
index_keep = np.logical_and(getattr(self.met_object, vrb) >= low, getattr(self.met_object, vrb) <= high)
|
|
229
|
+
self.met_object = self.filter_object_fields(self.met_object, self.met_fields, index_keep)
|
|
170
230
|
|
|
171
231
|
def block_data(
|
|
172
232
|
self, time_edges: pd.arrays.DatetimeArray, data_object: Union[SensorGroup, MeteorologyGroup]
|
|
@@ -229,9 +289,9 @@ class Preprocessor:
|
|
|
229
289
|
|
|
230
290
|
"""
|
|
231
291
|
return_object = deepcopy(data_object)
|
|
232
|
-
for
|
|
233
|
-
if getattr(return_object,
|
|
234
|
-
setattr(return_object,
|
|
292
|
+
for field_name in fields:
|
|
293
|
+
if getattr(return_object, field_name) is not None:
|
|
294
|
+
setattr(return_object, field_name, getattr(return_object, field_name)[index])
|
|
235
295
|
return return_object
|
|
236
296
|
|
|
237
297
|
def interpolate_single_met_object(self, met_in_object: Meteorology) -> Meteorology:
|
|
@@ -246,15 +306,15 @@ class Preprocessor:
|
|
|
246
306
|
"""
|
|
247
307
|
met_out_object = Meteorology()
|
|
248
308
|
time_out = None
|
|
249
|
-
for
|
|
250
|
-
if (
|
|
309
|
+
for field_name in self.met_fields:
|
|
310
|
+
if (field_name != "time") and (getattr(met_in_object, field_name) is not None):
|
|
251
311
|
time_out, resampled_values = temporal_resampling(
|
|
252
312
|
met_in_object.time,
|
|
253
|
-
getattr(met_in_object,
|
|
313
|
+
getattr(met_in_object, field_name),
|
|
254
314
|
self.time_bin_edges,
|
|
255
315
|
self.aggregate_function,
|
|
256
316
|
)
|
|
257
|
-
setattr(met_out_object,
|
|
317
|
+
setattr(met_out_object, field_name, resampled_values)
|
|
258
318
|
|
|
259
319
|
if time_out is not None:
|
|
260
320
|
met_out_object.time = time_out
|
pyelq/sensor/__init__.py
CHANGED
pyelq/sensor/sensor.py
CHANGED
|
@@ -107,6 +107,44 @@ class Sensor:
|
|
|
107
107
|
|
|
108
108
|
return fig
|
|
109
109
|
|
|
110
|
+
def subset_sensor(self, section_index: int) -> "Sensor":
|
|
111
|
+
"""Subset the sensor based on the provided section index.
|
|
112
|
+
|
|
113
|
+
The method is designed to return a new `Sensor` object containing only the observations corresponding to a
|
|
114
|
+
specified section index. Sections are defined by unique values in the `source_on` attribute. For a case where
|
|
115
|
+
the source is turned on and off multiple times, (0 values in `source_on` indicate off periods and positive
|
|
116
|
+
integers indicate different on periods). For example, if section_index=1, a new Sensor will be returned
|
|
117
|
+
containing only observations where self.source_on == 1.
|
|
118
|
+
|
|
119
|
+
If sensor.location.shape[0] and sensor.time.shape[0] align we assume the location values are dependent on time and
|
|
120
|
+
therefore need to be filtered accordingly and otherwise we keep the original location.
|
|
121
|
+
|
|
122
|
+
This functionality is useful for situations where data is collected in multiple sections, e.g. repeated on/off
|
|
123
|
+
releases where we want to work with one section at a time or later stitch multiple per-section segments
|
|
124
|
+
together.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
section_index (int): Integer indicating which observations to keep.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
new_sensor (Sensor): A new Sensor object containing only the specified observations.
|
|
131
|
+
|
|
132
|
+
"""
|
|
133
|
+
if self.source_on is None:
|
|
134
|
+
return deepcopy(self)
|
|
135
|
+
|
|
136
|
+
section_indices = (self.source_on == section_index).flatten()
|
|
137
|
+
new_sensor = deepcopy(self)
|
|
138
|
+
new_sensor.time = self.time[section_indices]
|
|
139
|
+
new_sensor.concentration = self.concentration[section_indices]
|
|
140
|
+
location_object = new_sensor.location.to_array()
|
|
141
|
+
if location_object.shape[0] == self.time.shape[0]:
|
|
142
|
+
location_object = location_object[section_indices, :]
|
|
143
|
+
new_sensor.location = new_sensor.location.from_array(location_object)
|
|
144
|
+
|
|
145
|
+
new_sensor.source_on = self.source_on[section_indices]
|
|
146
|
+
return new_sensor
|
|
147
|
+
|
|
110
148
|
|
|
111
149
|
@dataclass
|
|
112
150
|
class SensorGroup(dict):
|
|
@@ -170,7 +208,9 @@ class SensorGroup(dict):
|
|
|
170
208
|
|
|
171
209
|
@property
|
|
172
210
|
def source_on(self) -> np.ndarray:
|
|
173
|
-
"""Column vector of
|
|
211
|
+
"""Column vector of integers indicating whether sources are expected to be on, unwrapped over sensors.
|
|
212
|
+
|
|
213
|
+
Different integer values represent different sections where the source is on.
|
|
174
214
|
|
|
175
215
|
Assumes source is on when None is specified for a specific sensor.
|
|
176
216
|
|
|
@@ -179,14 +219,14 @@ class SensorGroup(dict):
|
|
|
179
219
|
|
|
180
220
|
"""
|
|
181
221
|
overall_idx = np.array([])
|
|
182
|
-
for curr_key in
|
|
222
|
+
for curr_key in self.keys():
|
|
183
223
|
if self[curr_key].source_on is None:
|
|
184
|
-
temp_idx = np.ones(self[curr_key].nof_observations).astype(
|
|
224
|
+
temp_idx = np.ones(self[curr_key].nof_observations).astype(int)
|
|
185
225
|
else:
|
|
186
|
-
temp_idx = self[curr_key].source_on
|
|
226
|
+
temp_idx = self[curr_key].source_on.flatten()
|
|
187
227
|
|
|
188
228
|
overall_idx = np.concatenate([overall_idx, temp_idx])
|
|
189
|
-
return overall_idx.astype(
|
|
229
|
+
return overall_idx.astype(int)
|
|
190
230
|
|
|
191
231
|
@property
|
|
192
232
|
def nof_sensors(self) -> int:
|
|
@@ -239,3 +279,28 @@ class SensorGroup(dict):
|
|
|
239
279
|
fig = sensor.plot_timeseries(fig, color=color_map[color_idx], mode=mode)
|
|
240
280
|
|
|
241
281
|
return fig
|
|
282
|
+
|
|
283
|
+
def subset_sensor(self, section_index: int) -> "SensorGroup":
|
|
284
|
+
"""Subset the sensor based on the provided section index.
|
|
285
|
+
|
|
286
|
+
The method is designed to return a new `SensorGroup` object containing only the observations corresponding to a
|
|
287
|
+
specified section index. Sections are defined by unique values in the `sensor.source_on` attribute. For a case
|
|
288
|
+
where the source is turned on and off multiple times, (0 values in `sensor.source_on` indicate off periods and
|
|
289
|
+
positive integers indicate different on periods). For example, if section_index=1, a new SensorGroup will be
|
|
290
|
+
returned containing only observations where sensor.source_on == 1.
|
|
291
|
+
This functionality is useful for situations where data is collected in multiple sections, e.g. repeated on/off
|
|
292
|
+
releases where we want to work with one section at a time or later stitch multiple per-section segments
|
|
293
|
+
together.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
section_index (int): An integer indicating which observations to keep.
|
|
297
|
+
|
|
298
|
+
Returns:
|
|
299
|
+
SensorGroup: A new SensorGroup object containing only the specified observations
|
|
300
|
+
|
|
301
|
+
"""
|
|
302
|
+
subset_sensor = SensorGroup()
|
|
303
|
+
for _, sensor in enumerate(self.values()):
|
|
304
|
+
subset_sensor_i = sensor.subset_sensor(section_index)
|
|
305
|
+
subset_sensor.add_sensor(subset_sensor_i)
|
|
306
|
+
return subset_sensor
|
pyelq/source_map.py
CHANGED