pycontrails 0.59.0__cp314-cp314-macosx_10_15_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pycontrails might be problematic. Click here for more details.
- pycontrails/__init__.py +70 -0
- pycontrails/_version.py +34 -0
- pycontrails/core/__init__.py +30 -0
- pycontrails/core/aircraft_performance.py +679 -0
- pycontrails/core/airports.py +228 -0
- pycontrails/core/cache.py +889 -0
- pycontrails/core/coordinates.py +174 -0
- pycontrails/core/fleet.py +483 -0
- pycontrails/core/flight.py +2185 -0
- pycontrails/core/flightplan.py +228 -0
- pycontrails/core/fuel.py +140 -0
- pycontrails/core/interpolation.py +702 -0
- pycontrails/core/met.py +2936 -0
- pycontrails/core/met_var.py +387 -0
- pycontrails/core/models.py +1321 -0
- pycontrails/core/polygon.py +549 -0
- pycontrails/core/rgi_cython.cpython-314-darwin.so +0 -0
- pycontrails/core/vector.py +2249 -0
- pycontrails/datalib/__init__.py +12 -0
- pycontrails/datalib/_met_utils/metsource.py +746 -0
- pycontrails/datalib/ecmwf/__init__.py +73 -0
- pycontrails/datalib/ecmwf/arco_era5.py +345 -0
- pycontrails/datalib/ecmwf/common.py +114 -0
- pycontrails/datalib/ecmwf/era5.py +554 -0
- pycontrails/datalib/ecmwf/era5_model_level.py +490 -0
- pycontrails/datalib/ecmwf/hres.py +804 -0
- pycontrails/datalib/ecmwf/hres_model_level.py +466 -0
- pycontrails/datalib/ecmwf/ifs.py +287 -0
- pycontrails/datalib/ecmwf/model_levels.py +435 -0
- pycontrails/datalib/ecmwf/static/model_level_dataframe_v20240418.csv +139 -0
- pycontrails/datalib/ecmwf/variables.py +268 -0
- pycontrails/datalib/geo_utils.py +261 -0
- pycontrails/datalib/gfs/__init__.py +28 -0
- pycontrails/datalib/gfs/gfs.py +656 -0
- pycontrails/datalib/gfs/variables.py +104 -0
- pycontrails/datalib/goes.py +764 -0
- pycontrails/datalib/gruan.py +343 -0
- pycontrails/datalib/himawari/__init__.py +27 -0
- pycontrails/datalib/himawari/header_struct.py +266 -0
- pycontrails/datalib/himawari/himawari.py +671 -0
- pycontrails/datalib/landsat.py +589 -0
- pycontrails/datalib/leo_utils/__init__.py +5 -0
- pycontrails/datalib/leo_utils/correction.py +266 -0
- pycontrails/datalib/leo_utils/landsat_metadata.py +300 -0
- pycontrails/datalib/leo_utils/search.py +250 -0
- pycontrails/datalib/leo_utils/sentinel_metadata.py +748 -0
- pycontrails/datalib/leo_utils/static/bq_roi_query.sql +6 -0
- pycontrails/datalib/leo_utils/vis.py +59 -0
- pycontrails/datalib/sentinel.py +650 -0
- pycontrails/datalib/spire/__init__.py +5 -0
- pycontrails/datalib/spire/exceptions.py +62 -0
- pycontrails/datalib/spire/spire.py +604 -0
- pycontrails/ext/bada.py +42 -0
- pycontrails/ext/cirium.py +14 -0
- pycontrails/ext/empirical_grid.py +140 -0
- pycontrails/ext/synthetic_flight.py +431 -0
- pycontrails/models/__init__.py +1 -0
- pycontrails/models/accf.py +425 -0
- pycontrails/models/apcemm/__init__.py +8 -0
- pycontrails/models/apcemm/apcemm.py +983 -0
- pycontrails/models/apcemm/inputs.py +226 -0
- pycontrails/models/apcemm/static/apcemm_yaml_template.yaml +183 -0
- pycontrails/models/apcemm/utils.py +437 -0
- pycontrails/models/cocip/__init__.py +29 -0
- pycontrails/models/cocip/cocip.py +2742 -0
- pycontrails/models/cocip/cocip_params.py +305 -0
- pycontrails/models/cocip/cocip_uncertainty.py +291 -0
- pycontrails/models/cocip/contrail_properties.py +1530 -0
- pycontrails/models/cocip/output_formats.py +2270 -0
- pycontrails/models/cocip/radiative_forcing.py +1260 -0
- pycontrails/models/cocip/radiative_heating.py +520 -0
- pycontrails/models/cocip/unterstrasser_wake_vortex.py +508 -0
- pycontrails/models/cocip/wake_vortex.py +396 -0
- pycontrails/models/cocip/wind_shear.py +120 -0
- pycontrails/models/cocipgrid/__init__.py +9 -0
- pycontrails/models/cocipgrid/cocip_grid.py +2552 -0
- pycontrails/models/cocipgrid/cocip_grid_params.py +138 -0
- pycontrails/models/dry_advection.py +602 -0
- pycontrails/models/emissions/__init__.py +21 -0
- pycontrails/models/emissions/black_carbon.py +599 -0
- pycontrails/models/emissions/emissions.py +1353 -0
- pycontrails/models/emissions/ffm2.py +336 -0
- pycontrails/models/emissions/static/default-engine-uids.csv +239 -0
- pycontrails/models/emissions/static/edb-gaseous-v29b-engines.csv +596 -0
- pycontrails/models/emissions/static/edb-nvpm-v29b-engines.csv +215 -0
- pycontrails/models/extended_k15.py +1327 -0
- pycontrails/models/humidity_scaling/__init__.py +37 -0
- pycontrails/models/humidity_scaling/humidity_scaling.py +1075 -0
- pycontrails/models/humidity_scaling/quantiles/era5-model-level-quantiles.pq +0 -0
- pycontrails/models/humidity_scaling/quantiles/era5-pressure-level-quantiles.pq +0 -0
- pycontrails/models/issr.py +210 -0
- pycontrails/models/pcc.py +326 -0
- pycontrails/models/pcr.py +154 -0
- pycontrails/models/ps_model/__init__.py +18 -0
- pycontrails/models/ps_model/ps_aircraft_params.py +381 -0
- pycontrails/models/ps_model/ps_grid.py +701 -0
- pycontrails/models/ps_model/ps_model.py +1000 -0
- pycontrails/models/ps_model/ps_operational_limits.py +525 -0
- pycontrails/models/ps_model/static/ps-aircraft-params-20250328.csv +69 -0
- pycontrails/models/ps_model/static/ps-synonym-list-20250328.csv +104 -0
- pycontrails/models/sac.py +442 -0
- pycontrails/models/tau_cirrus.py +183 -0
- pycontrails/physics/__init__.py +1 -0
- pycontrails/physics/constants.py +117 -0
- pycontrails/physics/geo.py +1138 -0
- pycontrails/physics/jet.py +968 -0
- pycontrails/physics/static/iata-cargo-load-factors-20250221.csv +74 -0
- pycontrails/physics/static/iata-passenger-load-factors-20250221.csv +74 -0
- pycontrails/physics/thermo.py +551 -0
- pycontrails/physics/units.py +472 -0
- pycontrails/py.typed +0 -0
- pycontrails/utils/__init__.py +1 -0
- pycontrails/utils/dependencies.py +66 -0
- pycontrails/utils/iteration.py +13 -0
- pycontrails/utils/json.py +187 -0
- pycontrails/utils/temp.py +50 -0
- pycontrails/utils/types.py +163 -0
- pycontrails-0.59.0.dist-info/METADATA +179 -0
- pycontrails-0.59.0.dist-info/RECORD +123 -0
- pycontrails-0.59.0.dist-info/WHEEL +6 -0
- pycontrails-0.59.0.dist-info/licenses/LICENSE +178 -0
- pycontrails-0.59.0.dist-info/licenses/NOTICE +43 -0
- pycontrails-0.59.0.dist-info/top_level.txt +3 -0
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
"""Support for Sentinel-2 imagery retrieval through Google Cloud Platform."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pathlib
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from xml.etree import ElementTree
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import numpy.typing as npt
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import xarray as xr
|
|
13
|
+
|
|
14
|
+
from pycontrails.core import Flight, cache
|
|
15
|
+
from pycontrails.datalib.leo_utils import correction, search
|
|
16
|
+
from pycontrails.datalib.leo_utils.sentinel_metadata import (
|
|
17
|
+
_band_id,
|
|
18
|
+
get_detector_id,
|
|
19
|
+
get_time_delay_detectors,
|
|
20
|
+
parse_ephemeris_sentinel,
|
|
21
|
+
parse_high_res_viewing_incidence_angles,
|
|
22
|
+
parse_sensing_time,
|
|
23
|
+
parse_sentinel_crs,
|
|
24
|
+
read_image_coordinates,
|
|
25
|
+
)
|
|
26
|
+
from pycontrails.datalib.leo_utils.vis import equalize, normalize
|
|
27
|
+
from pycontrails.utils import dependencies
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
import gcsfs
|
|
31
|
+
except ModuleNotFoundError as exc:
|
|
32
|
+
dependencies.raise_module_not_found_error(
|
|
33
|
+
name="sentinel module",
|
|
34
|
+
package_name="gcsfs",
|
|
35
|
+
module_not_found_error=exc,
|
|
36
|
+
pycontrails_optional_package="sat",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
import pyproj
|
|
41
|
+
except ModuleNotFoundError as exc:
|
|
42
|
+
dependencies.raise_module_not_found_error(
|
|
43
|
+
name="sentinel module",
|
|
44
|
+
package_name="pyproj",
|
|
45
|
+
module_not_found_error=exc,
|
|
46
|
+
pycontrails_optional_package="sat",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
from PIL import Image
|
|
51
|
+
except ModuleNotFoundError as exc:
|
|
52
|
+
dependencies.raise_module_not_found_error(
|
|
53
|
+
name="sentinel module",
|
|
54
|
+
package_name="pillow",
|
|
55
|
+
module_not_found_error=exc,
|
|
56
|
+
pycontrails_optional_package="sat",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_path_to_static = pathlib.Path(__file__).parent / "static"
|
|
61
|
+
ROI_QUERY_FILENAME = _path_to_static / "sentinel_roi_query.sql"
|
|
62
|
+
|
|
63
|
+
#: BigQuery table with imagery metadata
|
|
64
|
+
BQ_TABLE = "bigquery-public-data.cloud_storage_geo_index.sentinel_2_index"
|
|
65
|
+
|
|
66
|
+
#: Default columns to include in queries
|
|
67
|
+
BQ_DEFAULT_COLUMNS = ["base_url", "granule_id", "sensing_time", "source_url"]
|
|
68
|
+
|
|
69
|
+
#: Default spatial extent for queries
|
|
70
|
+
BQ_DEFAULT_EXTENT = search.GLOBAL_EXTENT
|
|
71
|
+
|
|
72
|
+
#: Default Sentinel channels to use if none are specified.
|
|
73
|
+
#: These are visible bands for producing a true color composite.
|
|
74
|
+
DEFAULT_BANDS = ["B02", "B03", "B04"]
|
|
75
|
+
|
|
76
|
+
#: Strip this prefix from GCP URLs when caching Sentinel files locally
|
|
77
|
+
GCP_STRIP_PREFIX = "gs://gcp-public-data-sentinel-2/"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def query(
|
|
81
|
+
start_time: np.datetime64,
|
|
82
|
+
end_time: np.datetime64,
|
|
83
|
+
extent: str | None = None,
|
|
84
|
+
columns: list[str] | None = None,
|
|
85
|
+
) -> pd.DataFrame:
|
|
86
|
+
"""Find Sentinel-2 imagery within spatiotemporal region of interest.
|
|
87
|
+
|
|
88
|
+
This function requires access to the
|
|
89
|
+
`Google BigQuery API <https://cloud.google.com/bigquery?hl=en>`__
|
|
90
|
+
and uses the `BigQuery python library <https://cloud.google.com/python/docs/reference/bigquery/latest/index.html>`__.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
start_time : np.datetime64
|
|
95
|
+
Start of time period for search
|
|
96
|
+
end_time : np.datetime64
|
|
97
|
+
End of time period for search
|
|
98
|
+
extent : str | None, optional
|
|
99
|
+
Spatial region of interest as a GeoJSON string. If not provided, defaults
|
|
100
|
+
to a global extent.
|
|
101
|
+
columns : list[str] | None, optional
|
|
102
|
+
Columns to return from Google
|
|
103
|
+
`BigQuery table <https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=cloud_storage_geo_index&t=landsat_index&page=table&_ga=2.90807450.1051800793.1716904050-255800408.1705955196>`__.
|
|
104
|
+
By default, returns imagery base URL, granule ID, and sensing time.
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
pd.DataFrame
|
|
109
|
+
Query results in pandas DataFrame
|
|
110
|
+
|
|
111
|
+
See Also
|
|
112
|
+
--------
|
|
113
|
+
:func:`search.query`
|
|
114
|
+
"""
|
|
115
|
+
extent = extent or BQ_DEFAULT_EXTENT
|
|
116
|
+
roi = search.ROI(start_time, end_time, extent)
|
|
117
|
+
columns = columns or BQ_DEFAULT_COLUMNS
|
|
118
|
+
return search.query(BQ_TABLE, roi, columns)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def intersect(
|
|
122
|
+
flight: Flight,
|
|
123
|
+
columns: list[str] | None = None,
|
|
124
|
+
) -> pd.DataFrame:
|
|
125
|
+
"""Find Sentinel-2 imagery intersecting with flight track.
|
|
126
|
+
|
|
127
|
+
This function will return all scenes with a bounding box that includes flight waypoints
|
|
128
|
+
both before and after the sensing time.
|
|
129
|
+
|
|
130
|
+
This function requires access to the
|
|
131
|
+
`Google BigQuery API <https://cloud.google.com/bigquery?hl=en>`__
|
|
132
|
+
and uses the `BigQuery python library <https://cloud.google.com/python/docs/reference/bigquery/latest/index.html>`__.
|
|
133
|
+
|
|
134
|
+
Parameters
|
|
135
|
+
----------
|
|
136
|
+
flight : Flight
|
|
137
|
+
Flight for intersection
|
|
138
|
+
columns : list[str] | None, optional
|
|
139
|
+
Columns to return from Google
|
|
140
|
+
`BigQuery table <https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=cloud_storage_geo_index&t=landsat_index&page=table&_ga=2.90807450.1051800793.1716904050-255800408.1705955196>`__.
|
|
141
|
+
By default, returns imagery base URL, granule ID, and sensing time.
|
|
142
|
+
|
|
143
|
+
Returns
|
|
144
|
+
-------
|
|
145
|
+
pd.DataFrame
|
|
146
|
+
Query results in pandas DataFrame
|
|
147
|
+
|
|
148
|
+
See Also
|
|
149
|
+
--------
|
|
150
|
+
:func:`search.intersect`
|
|
151
|
+
"""
|
|
152
|
+
columns = columns or BQ_DEFAULT_COLUMNS
|
|
153
|
+
scenes = search.intersect(BQ_TABLE, flight, columns)
|
|
154
|
+
|
|
155
|
+
# overwrite the base_url with source_url.
|
|
156
|
+
# After 2024-03-14 there is a mistake in the Google BigQuery table,
|
|
157
|
+
# such that the base_url is written to the source_url column
|
|
158
|
+
scenes["base_url"] = scenes["base_url"].fillna(scenes["source_url"])
|
|
159
|
+
return scenes
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class Sentinel:
|
|
163
|
+
"""Support for Sentinel-2 data handling.
|
|
164
|
+
|
|
165
|
+
Parameters
|
|
166
|
+
----------
|
|
167
|
+
base_url : str
|
|
168
|
+
Base URL of Sentinel-2 scene. To find URLs for Sentinel-2 scenes at
|
|
169
|
+
specific locations and times, see :func:`query` and :func:`intersect`.
|
|
170
|
+
granule_id : str
|
|
171
|
+
Granule ID of Sentinel-2 scene. To find URLs for Sentinel-2 scenes at
|
|
172
|
+
specific locations and times, see :func:`query` and :func:`intersect`.
|
|
173
|
+
bands : str | Iterable[str] | None
|
|
174
|
+
Set of bands to retrieve. The 13 possible bands are represented by
|
|
175
|
+
the string "B01" to "B12" plus "B8A". For the true color scheme, set
|
|
176
|
+
``bands=("B02", "B03", "B04")``. By default, bands for the true color scheme
|
|
177
|
+
are used. Bands must share a common resolution. The resolutions of each band are:
|
|
178
|
+
|
|
179
|
+
- B02-B04, B08: 10 m
|
|
180
|
+
- B05-B07, B8A, B11, B12: 20 m
|
|
181
|
+
- B01, B09, B10: 60 m
|
|
182
|
+
|
|
183
|
+
cachestore : cache.CacheStore | None, optional
|
|
184
|
+
Cache store for Landsat data. If None, a :class:`DiskCacheStore` is used.
|
|
185
|
+
|
|
186
|
+
See Also
|
|
187
|
+
--------
|
|
188
|
+
query
|
|
189
|
+
intersect
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
def __init__(
|
|
193
|
+
self,
|
|
194
|
+
base_url: str,
|
|
195
|
+
granule_id: str,
|
|
196
|
+
bands: str | Iterable[str] | None = None,
|
|
197
|
+
cachestore: cache.CacheStore | None = None,
|
|
198
|
+
) -> None:
|
|
199
|
+
self.base_url = base_url
|
|
200
|
+
self.granule_id = granule_id
|
|
201
|
+
self.bands = _parse_bands(bands)
|
|
202
|
+
_check_band_resolution(self.bands)
|
|
203
|
+
self.fs = gcsfs.GCSFileSystem(token="anon")
|
|
204
|
+
|
|
205
|
+
if cachestore is None:
|
|
206
|
+
cache_root = cache._get_user_cache_dir()
|
|
207
|
+
cache_dir = f"{cache_root}/sentinel"
|
|
208
|
+
cachestore = cache.DiskCacheStore(cache_dir=cache_dir)
|
|
209
|
+
self.cachestore = cachestore
|
|
210
|
+
|
|
211
|
+
def __repr__(self) -> str:
|
|
212
|
+
"""Return string representation."""
|
|
213
|
+
return (
|
|
214
|
+
f"Sentinel(base_url='{self.base_url}',\n"
|
|
215
|
+
f"\tgranule_id='{self.granule_id}',\n"
|
|
216
|
+
f"\tbands={sorted(self.bands)})"
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
def get(self, reflective: str = "reflectance") -> xr.Dataset:
|
|
220
|
+
"""Retrieve Sentinel-2 imagery.
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
reflective : str, optional
|
|
225
|
+
Set to "raw" to return raw values or "reflectance" for rescaled reflectances.
|
|
226
|
+
By default, return reflectances.
|
|
227
|
+
|
|
228
|
+
Returns
|
|
229
|
+
-------
|
|
230
|
+
xr.Dataset
|
|
231
|
+
Dataset of Sentinel-2 data.
|
|
232
|
+
"""
|
|
233
|
+
available = ("raw", "reflectance")
|
|
234
|
+
if reflective not in available:
|
|
235
|
+
msg = f"reflective band processing must be one of {available}"
|
|
236
|
+
raise ValueError(msg)
|
|
237
|
+
|
|
238
|
+
data = {band: self._get(band, reflective) for band in self.bands}
|
|
239
|
+
return xr.Dataset(data)
|
|
240
|
+
|
|
241
|
+
def get_viewing_angle_metadata(self, scale: int = 10) -> xr.Dataset:
|
|
242
|
+
"""Return the dataset with viewing angles.
|
|
243
|
+
|
|
244
|
+
See :func:`parse_high_res_viewing_incidence_angles` for details.
|
|
245
|
+
"""
|
|
246
|
+
granule_meta_path, _ = self._get_meta()
|
|
247
|
+
_, detector_band_path = self._get_correction_meta()
|
|
248
|
+
return parse_high_res_viewing_incidence_angles(
|
|
249
|
+
granule_meta_path, detector_band_path, scale=scale
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def get_detector_id(
|
|
253
|
+
self, x: npt.NDArray[np.floating], y: npt.NDArray[np.floating]
|
|
254
|
+
) -> npt.NDArray[np.integer]:
|
|
255
|
+
"""Return the detector_id of the Sentinel-2 detector that imaged the given points.
|
|
256
|
+
|
|
257
|
+
Parameters
|
|
258
|
+
----------
|
|
259
|
+
x : npt.NDArray[np.floating]
|
|
260
|
+
x coordinates of points in the Sentinel-2 image CRS
|
|
261
|
+
y : npt.NDArray[np.floating]
|
|
262
|
+
y coordinates of points in the Sentinel-2 image CRS
|
|
263
|
+
|
|
264
|
+
Returns
|
|
265
|
+
-------
|
|
266
|
+
npt.NDArray[np.integer]
|
|
267
|
+
Detector IDs for each point. If a point is outside the image, the detector ID is 0.
|
|
268
|
+
"""
|
|
269
|
+
granule_sink, _ = self._get_meta()
|
|
270
|
+
_, detector_band_sink = self._get_correction_meta()
|
|
271
|
+
return get_detector_id(detector_band_sink, granule_sink, x, y)
|
|
272
|
+
|
|
273
|
+
def get_time_delay_detector(
|
|
274
|
+
self, detector_id: npt.NDArray[np.integer], band: str = "B03"
|
|
275
|
+
) -> npt.NDArray[np.timedelta64]:
|
|
276
|
+
"""Return the time delay for the given detector IDs."""
|
|
277
|
+
datastrip_sink, _ = self._get_correction_meta()
|
|
278
|
+
delays = get_time_delay_detectors(datastrip_sink, band)
|
|
279
|
+
|
|
280
|
+
# Map the delays to the input array
|
|
281
|
+
unique_ids, inverse = np.unique(detector_id, return_inverse=True)
|
|
282
|
+
|
|
283
|
+
# Get offsets for each unique detector
|
|
284
|
+
offset_list = []
|
|
285
|
+
for did in unique_ids:
|
|
286
|
+
if did == 0:
|
|
287
|
+
offset_list.append(np.timedelta64("NaT"))
|
|
288
|
+
else:
|
|
289
|
+
offset_list.append(delays.get(did, np.timedelta64("NaT")))
|
|
290
|
+
offset_values = np.array(offset_list, dtype="timedelta64[ns]")
|
|
291
|
+
|
|
292
|
+
# Map back to the original array
|
|
293
|
+
return offset_values[inverse]
|
|
294
|
+
|
|
295
|
+
def get_ephemeris(self) -> pd.DataFrame:
|
|
296
|
+
"""Return the satellite ephemeris as a :class:`pandas.DataFrame`."""
|
|
297
|
+
datastrip_sink, _ = self._get_correction_meta()
|
|
298
|
+
|
|
299
|
+
return parse_ephemeris_sentinel(datastrip_sink)
|
|
300
|
+
|
|
301
|
+
def get_crs(self) -> pyproj.CRS:
|
|
302
|
+
"""Return the CRS of the satellite image."""
|
|
303
|
+
granule_meta_path, _ = self._get_meta()
|
|
304
|
+
return parse_sentinel_crs(granule_meta_path)
|
|
305
|
+
|
|
306
|
+
def get_sensing_time(self) -> pd.Timestamp:
|
|
307
|
+
"""Return the sensing_time of the satellite image."""
|
|
308
|
+
granule_meta_path, _ = self._get_meta()
|
|
309
|
+
return parse_sensing_time(granule_meta_path)
|
|
310
|
+
|
|
311
|
+
def colocate_flight(
|
|
312
|
+
self,
|
|
313
|
+
flight: Flight,
|
|
314
|
+
band: str = "B03",
|
|
315
|
+
) -> tuple[float, float, np.datetime64]:
|
|
316
|
+
"""Colocate a flight track with satellite image pixels.
|
|
317
|
+
|
|
318
|
+
This function first projects flight waypoints into the UTM coordinate system
|
|
319
|
+
of the satellite image, then applies a viewing angle correction to estimate
|
|
320
|
+
the actual ground position imaged by the satellite. Next, the scan time for
|
|
321
|
+
each point is estimated based on the satellite meta data. Finally, the
|
|
322
|
+
point along the flight track is found where the flight time matches the
|
|
323
|
+
satellite scan time.
|
|
324
|
+
|
|
325
|
+
Parameters
|
|
326
|
+
----------
|
|
327
|
+
flight : Flight
|
|
328
|
+
The flight object.
|
|
329
|
+
band : str, optional
|
|
330
|
+
Spectral band to use for geometry parsing. Default is "B03".
|
|
331
|
+
|
|
332
|
+
Returns
|
|
333
|
+
-------
|
|
334
|
+
tuple[float, float, np.datetime64]
|
|
335
|
+
A tuple containing the x and y coordinates of the flight position in the
|
|
336
|
+
satellite image CRS, and the corrected sensing_time of the satellite image.
|
|
337
|
+
"""
|
|
338
|
+
utm_crs = self.get_crs()
|
|
339
|
+
|
|
340
|
+
# Project from WGS84 to the x and y coordinates in the UTM coordinate system
|
|
341
|
+
transformer = pyproj.Transformer.from_crs("EPSG:4326", utm_crs, always_xy=True)
|
|
342
|
+
x, y = transformer.transform(flight["longitude"], flight["latitude"])
|
|
343
|
+
z = flight.altitude
|
|
344
|
+
|
|
345
|
+
# Apply sensing angle correction
|
|
346
|
+
ds_angles = self.get_viewing_angle_metadata()
|
|
347
|
+
x_proj, y_proj = correction.scan_angle_correction(ds_angles, x, y, z, maxiter=3)
|
|
348
|
+
|
|
349
|
+
# Get the satellite ephemeris data prepared
|
|
350
|
+
ephemeris = self.get_ephemeris()
|
|
351
|
+
|
|
352
|
+
# Estimate the scan time for each point
|
|
353
|
+
scan_time_no_offset = correction.estimate_scan_time(ephemeris, utm_crs, x_proj, y_proj)
|
|
354
|
+
|
|
355
|
+
detector_id = self.get_detector_id(x_proj, y_proj)
|
|
356
|
+
offset = self.get_time_delay_detector(detector_id, band=band)
|
|
357
|
+
scan_time = scan_time_no_offset + offset
|
|
358
|
+
|
|
359
|
+
# To finish up, find the point along the project flight track at which the satellite
|
|
360
|
+
# scan time equals the flight time
|
|
361
|
+
# It's possible that if the flight has a really fine time resolution, that
|
|
362
|
+
# the diff_times array will have multiple zeros. This is okay, as np.interp
|
|
363
|
+
# will just return the first one.
|
|
364
|
+
valid = np.isfinite(x_proj) & np.isfinite(y_proj) & np.isfinite(scan_time)
|
|
365
|
+
|
|
366
|
+
diff_times = flight["time"][valid] - scan_time[valid]
|
|
367
|
+
if diff_times[0] > np.timedelta64(0) or diff_times[-1] < np.timedelta64(0):
|
|
368
|
+
msg = "Flight track does not overlap with satellite image in time."
|
|
369
|
+
raise ValueError(msg)
|
|
370
|
+
|
|
371
|
+
index = np.arange(len(diff_times))
|
|
372
|
+
frac_idx = np.interp(0, diff_times.astype(int), index)
|
|
373
|
+
|
|
374
|
+
x1 = np.interp(frac_idx, index, x_proj[valid]).item()
|
|
375
|
+
y1 = np.interp(frac_idx, index, y_proj[valid]).item()
|
|
376
|
+
t1 = np.interp(frac_idx, index, scan_time[valid].astype(int)).item()
|
|
377
|
+
|
|
378
|
+
return x1, y1, np.datetime64(int(t1), "ns")
|
|
379
|
+
|
|
380
|
+
# -------------------------------------------------------------------------------------
|
|
381
|
+
|
|
382
|
+
def _get(self, band: str, processing: str) -> xr.DataArray:
|
|
383
|
+
"""Download Sentinel-2 band to the :attr:`cachestore` and return processed data."""
|
|
384
|
+
jp2_path = self._get_jp2(band)
|
|
385
|
+
granule_meta_path, safe_meta_path = self._get_meta()
|
|
386
|
+
return _read(jp2_path, granule_meta_path, safe_meta_path, band, processing)
|
|
387
|
+
|
|
388
|
+
def _get_jp2(self, band: str) -> str:
|
|
389
|
+
"""Download Sentinel-2 imagery and return path to cached file."""
|
|
390
|
+
fs = self.fs
|
|
391
|
+
base_url = self.base_url
|
|
392
|
+
granule_id = self.granule_id
|
|
393
|
+
prefix = f"{base_url}/GRANULE/{granule_id}/IMG_DATA"
|
|
394
|
+
files = fs.ls(prefix)
|
|
395
|
+
|
|
396
|
+
urls = [f"gs://{f}" for f in files if f.endswith(f"{band}.jp2")]
|
|
397
|
+
if len(urls) > 1:
|
|
398
|
+
msg = f"Multiple image files found for band {band}"
|
|
399
|
+
raise ValueError(msg)
|
|
400
|
+
if len(urls) == 0:
|
|
401
|
+
msg = f"No image files found for band {band}"
|
|
402
|
+
raise ValueError(msg)
|
|
403
|
+
url = urls[0]
|
|
404
|
+
|
|
405
|
+
sink = self.cachestore.path(url.removeprefix(GCP_STRIP_PREFIX))
|
|
406
|
+
if not self.cachestore.exists(sink):
|
|
407
|
+
fs.get(url, sink)
|
|
408
|
+
return sink
|
|
409
|
+
|
|
410
|
+
def _get_meta(self) -> tuple[str, str]:
|
|
411
|
+
"""Download Sentinel-2 metadata files and return path to cached files.
|
|
412
|
+
|
|
413
|
+
Note that two XML files must be retrieved: one inside the GRANULE
|
|
414
|
+
subdirectory, and one at the top level of the SAFE archive.
|
|
415
|
+
"""
|
|
416
|
+
fs = self.fs
|
|
417
|
+
base_url = self.base_url
|
|
418
|
+
granule_id = self.granule_id
|
|
419
|
+
|
|
420
|
+
url = f"{base_url}/GRANULE/{granule_id}/MTD_TL.xml"
|
|
421
|
+
granule_sink = self.cachestore.path(url.removeprefix(GCP_STRIP_PREFIX))
|
|
422
|
+
if not self.cachestore.exists(granule_sink):
|
|
423
|
+
fs.get(url, granule_sink)
|
|
424
|
+
|
|
425
|
+
url = f"{base_url}/MTD_MSIL1C.xml"
|
|
426
|
+
safe_sink = self.cachestore.path(url.removeprefix(GCP_STRIP_PREFIX))
|
|
427
|
+
if not self.cachestore.exists(safe_sink):
|
|
428
|
+
fs.get(url, safe_sink)
|
|
429
|
+
|
|
430
|
+
return granule_sink, safe_sink
|
|
431
|
+
|
|
432
|
+
def _get_correction_meta(self) -> tuple[str, str]:
|
|
433
|
+
"""Download Sentinel-2 metadata files and return path to cached files.
|
|
434
|
+
|
|
435
|
+
Note that two XML files must be retrieved: one inside the GRANULE
|
|
436
|
+
subdirectory, and one at the top level of the SAFE archive.
|
|
437
|
+
"""
|
|
438
|
+
fs = self.fs
|
|
439
|
+
base_url = self.base_url
|
|
440
|
+
granule_id = self.granule_id
|
|
441
|
+
|
|
442
|
+
# Resolve the unknown subfolder in DATASTRIP using glob
|
|
443
|
+
# Probably there is a better method, but this worked for now
|
|
444
|
+
pattern = f"{base_url}/DATASTRIP/*"
|
|
445
|
+
matches = fs.glob(pattern)
|
|
446
|
+
if not matches:
|
|
447
|
+
raise FileNotFoundError(f"No DATASTRIP MTD_MDS.xml file found at pattern: {pattern}")
|
|
448
|
+
if len(matches) > 2:
|
|
449
|
+
raise RuntimeError(f"Multiple DATASTRIP MTD_MDS.xml files found: {matches}")
|
|
450
|
+
|
|
451
|
+
datastrip_id = matches[0].split("/")[-1].replace("_$folder$", "")
|
|
452
|
+
url = f"{base_url}/DATASTRIP/{datastrip_id}/MTD_DS.xml"
|
|
453
|
+
datastrip_sink = self.cachestore.path(url.removeprefix(GCP_STRIP_PREFIX))
|
|
454
|
+
if not self.cachestore.exists(datastrip_sink):
|
|
455
|
+
fs.get(url, datastrip_sink)
|
|
456
|
+
|
|
457
|
+
# Path to the QI_DATA folder
|
|
458
|
+
qi_data_url = f"{base_url}/GRANULE/{granule_id}/QI_DATA/"
|
|
459
|
+
|
|
460
|
+
# List files in QI_DATA (assuming fs can list directories)
|
|
461
|
+
files = fs.ls(qi_data_url)
|
|
462
|
+
|
|
463
|
+
# Filter for the detector mask files
|
|
464
|
+
mask_files = [f for f in files if "MSK_DETFOO_B03" in f]
|
|
465
|
+
|
|
466
|
+
if not mask_files:
|
|
467
|
+
raise FileNotFoundError(f"No detector mask found in {qi_data_url}")
|
|
468
|
+
|
|
469
|
+
# Choose the first available file (could prioritize .jp2 over .gml if needed)
|
|
470
|
+
mask_file = None
|
|
471
|
+
for ext in [".jp2", ".gml"]:
|
|
472
|
+
for f in mask_files:
|
|
473
|
+
if f.endswith(ext):
|
|
474
|
+
mask_file = f
|
|
475
|
+
break
|
|
476
|
+
if mask_file:
|
|
477
|
+
break
|
|
478
|
+
|
|
479
|
+
if mask_file is not None:
|
|
480
|
+
# Path where the file will be cached
|
|
481
|
+
detector_band_sink = self.cachestore.path(mask_file.removeprefix(GCP_STRIP_PREFIX))
|
|
482
|
+
if not self.cachestore.exists(detector_band_sink):
|
|
483
|
+
fs.get(mask_file, detector_band_sink)
|
|
484
|
+
|
|
485
|
+
return datastrip_sink, detector_band_sink
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _parse_bands(bands: str | Iterable[str] | None) -> set[str]:
|
|
489
|
+
"""Check that the bands are valid and return as a set."""
|
|
490
|
+
if bands is None:
|
|
491
|
+
return set(DEFAULT_BANDS)
|
|
492
|
+
|
|
493
|
+
if isinstance(bands, str):
|
|
494
|
+
bands = (bands,)
|
|
495
|
+
|
|
496
|
+
available = {f"B{i:02d}" for i in range(1, 13)} | {"B8A"}
|
|
497
|
+
bands = {b.upper() for b in bands}
|
|
498
|
+
if len(bands) == 0:
|
|
499
|
+
msg = "At least one band must be provided"
|
|
500
|
+
raise ValueError(msg)
|
|
501
|
+
if not bands.issubset(available):
|
|
502
|
+
msg = f"Bands must be in {sorted(available)}"
|
|
503
|
+
raise ValueError(msg)
|
|
504
|
+
return bands
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _check_band_resolution(bands: set[str]) -> None:
|
|
508
|
+
"""Confirm requested bands have a common horizontal resolution."""
|
|
509
|
+
groups = [
|
|
510
|
+
{"B02", "B03", "B04", "B08"}, # 10 m
|
|
511
|
+
{"B05", "B06", "B07", "B8A", "B11", "B12"}, # 20 m
|
|
512
|
+
{"B01", "B09", "B10"}, # 60 m
|
|
513
|
+
]
|
|
514
|
+
if not any(bands.issubset(group) for group in groups):
|
|
515
|
+
msg = "Bands must have a common horizontal resolution."
|
|
516
|
+
raise ValueError(msg)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _read(path: str, granule_meta: str, safe_meta: str, band: str, processing: str) -> xr.DataArray:
|
|
520
|
+
"""Read imagery data from Sentinel-2 files."""
|
|
521
|
+
Image.MAX_IMAGE_PIXELS = None # avoid decompression bomb warning
|
|
522
|
+
with Image.open(path) as src:
|
|
523
|
+
img = np.asarray(src)
|
|
524
|
+
|
|
525
|
+
if processing == "reflectance":
|
|
526
|
+
gain, offset = _read_band_reflectance_rescaling(safe_meta, band)
|
|
527
|
+
img = np.where(img == 0, np.nan, (img + offset) / gain).astype("float32")
|
|
528
|
+
|
|
529
|
+
tree = ElementTree.parse(granule_meta)
|
|
530
|
+
elem = tree.find(".//HORIZONTAL_CS_CODE")
|
|
531
|
+
if elem is None or elem.text is None:
|
|
532
|
+
msg = "Could not find imagery projection in metadata."
|
|
533
|
+
raise ValueError(msg)
|
|
534
|
+
epsg = int(elem.text.split(":")[1])
|
|
535
|
+
crs = pyproj.CRS.from_epsg(epsg)
|
|
536
|
+
|
|
537
|
+
x, y = read_image_coordinates(granule_meta, band)
|
|
538
|
+
|
|
539
|
+
da = xr.DataArray(
|
|
540
|
+
data=img,
|
|
541
|
+
coords={"y": y, "x": x},
|
|
542
|
+
dims=("y", "x"),
|
|
543
|
+
attrs={
|
|
544
|
+
"long_name": f"{band} {processing}",
|
|
545
|
+
"units": "nondim" if processing == "reflectance" else "none",
|
|
546
|
+
"crs": crs,
|
|
547
|
+
},
|
|
548
|
+
)
|
|
549
|
+
da["x"].attrs = {"long_name": "easting", "units": "m"}
|
|
550
|
+
da["y"].attrs = {"long_name": "northing", "units": "m"}
|
|
551
|
+
return da
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _read_band_reflectance_rescaling(meta: str, band: str) -> tuple[float, float]:
|
|
555
|
+
"""Read reflectance rescaling factors from metadata file.
|
|
556
|
+
|
|
557
|
+
See https://sentiwiki.copernicus.eu/web/s2-processing#S2Processing-TOAReflectanceComputation
|
|
558
|
+
and https://scihub.copernicus.eu/news/News00931.
|
|
559
|
+
"""
|
|
560
|
+
# Find quantization gain (present in all files)
|
|
561
|
+
tree = ElementTree.parse(meta)
|
|
562
|
+
elem = tree.find(".//QUANTIFICATION_VALUE")
|
|
563
|
+
if elem is None or elem.text is None:
|
|
564
|
+
msg = "Could not find reflectance quantization gain."
|
|
565
|
+
raise ValueError(msg)
|
|
566
|
+
gain = float(elem.text)
|
|
567
|
+
|
|
568
|
+
# See if offset (used in recently processed files) is present
|
|
569
|
+
elems = tree.findall(".//RADIO_ADD_OFFSET")
|
|
570
|
+
|
|
571
|
+
# If not, set offset to 0
|
|
572
|
+
if len(elems) == 0:
|
|
573
|
+
return gain, 0.0
|
|
574
|
+
|
|
575
|
+
# Otherwise, search for offset with correct band ID
|
|
576
|
+
band_id = _band_id(band)
|
|
577
|
+
for elem in elems:
|
|
578
|
+
if int(elem.attrib["band_id"]) == band_id and elem.text is not None:
|
|
579
|
+
offset = float(elem.text)
|
|
580
|
+
return gain, offset
|
|
581
|
+
|
|
582
|
+
msg = f"Could not find reflectance offset for band {band} (band ID {band_id})"
|
|
583
|
+
raise ValueError(msg)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def extract_sentinel_visualization(
|
|
587
|
+
ds: xr.Dataset, color_scheme: str = "true"
|
|
588
|
+
) -> tuple[npt.NDArray[np.float32], pyproj.CRS, tuple[float, float, float, float]]:
|
|
589
|
+
"""Extract artifacts for visualizing Sentinel data with the given color scheme.
|
|
590
|
+
|
|
591
|
+
Parameters
|
|
592
|
+
----------
|
|
593
|
+
ds : xr.Dataset
|
|
594
|
+
Dataset of Sentinel data as returned by :meth:`Sentinel.get`.
|
|
595
|
+
color_scheme : str, optional
|
|
596
|
+
Color scheme to use for visualization. The true color scheme
|
|
597
|
+
(`"true"`, the only option currently implemented) requires bands B02, B03, and B04.
|
|
598
|
+
|
|
599
|
+
Returns
|
|
600
|
+
-------
|
|
601
|
+
rgb : npt.NDArray[np.float32]
|
|
602
|
+
3D RGB array of shape ``(height, width, 3)``.
|
|
603
|
+
src_crs : pyproj.CRS
|
|
604
|
+
Imagery projection
|
|
605
|
+
src_extent : tuple[float, float, float, float]
|
|
606
|
+
Imagery extent in projected coordinates
|
|
607
|
+
"""
|
|
608
|
+
|
|
609
|
+
if color_scheme == "true":
|
|
610
|
+
rgb, src_crs = to_true_color(ds)
|
|
611
|
+
else:
|
|
612
|
+
raise ValueError(f"Color scheme must be 'true', not '{color_scheme}'")
|
|
613
|
+
|
|
614
|
+
x = ds["x"].values
|
|
615
|
+
y = ds["y"].values
|
|
616
|
+
src_extent = x.min(), x.max(), y.min(), y.max()
|
|
617
|
+
|
|
618
|
+
return rgb, src_crs, src_extent
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def to_true_color(ds: xr.Dataset) -> tuple[np.ndarray, pyproj.CRS]:
|
|
622
|
+
"""Compute 3d RGB array for the true color scheme.
|
|
623
|
+
|
|
624
|
+
Parameters
|
|
625
|
+
----------
|
|
626
|
+
ds : xr.Dataset
|
|
627
|
+
DataArray of Sentinel data with bands B02, B03, and B04.
|
|
628
|
+
|
|
629
|
+
Returns
|
|
630
|
+
-------
|
|
631
|
+
np.ndarray
|
|
632
|
+
3d RGB array with true color scheme.
|
|
633
|
+
|
|
634
|
+
src_crs : pyproj.CRS
|
|
635
|
+
Imagery projection
|
|
636
|
+
"""
|
|
637
|
+
red = ds["B04"]
|
|
638
|
+
green = ds["B03"]
|
|
639
|
+
blue = ds["B02"]
|
|
640
|
+
|
|
641
|
+
crs = red.attrs["crs"]
|
|
642
|
+
if not (crs.equals(green.attrs["crs"]) and crs.equals(blue.attrs["crs"])):
|
|
643
|
+
msg = "Bands B02, B03, and B04 do not share a common projection."
|
|
644
|
+
raise ValueError(msg)
|
|
645
|
+
|
|
646
|
+
img = np.dstack(
|
|
647
|
+
[equalize(normalize(band.values), clip_limit=0.03) for band in (red, green, blue)]
|
|
648
|
+
)
|
|
649
|
+
|
|
650
|
+
return img, crs
|