rslearn 0.0.1__py3-none-any.whl → 0.0.21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. rslearn/arg_parser.py +31 -0
  2. rslearn/config/__init__.py +6 -12
  3. rslearn/config/dataset.py +520 -401
  4. rslearn/const.py +9 -15
  5. rslearn/data_sources/__init__.py +8 -23
  6. rslearn/data_sources/aws_landsat.py +242 -98
  7. rslearn/data_sources/aws_open_data.py +111 -151
  8. rslearn/data_sources/aws_sentinel1.py +131 -0
  9. rslearn/data_sources/climate_data_store.py +471 -0
  10. rslearn/data_sources/copernicus.py +884 -12
  11. rslearn/data_sources/data_source.py +43 -12
  12. rslearn/data_sources/earthdaily.py +484 -0
  13. rslearn/data_sources/earthdata_srtm.py +282 -0
  14. rslearn/data_sources/eurocrops.py +242 -0
  15. rslearn/data_sources/gcp_public_data.py +578 -222
  16. rslearn/data_sources/google_earth_engine.py +461 -135
  17. rslearn/data_sources/local_files.py +219 -150
  18. rslearn/data_sources/openstreetmap.py +51 -89
  19. rslearn/data_sources/planet.py +24 -60
  20. rslearn/data_sources/planet_basemap.py +275 -0
  21. rslearn/data_sources/planetary_computer.py +798 -0
  22. rslearn/data_sources/usda_cdl.py +195 -0
  23. rslearn/data_sources/usgs_landsat.py +115 -83
  24. rslearn/data_sources/utils.py +249 -61
  25. rslearn/data_sources/vector_source.py +1 -0
  26. rslearn/data_sources/worldcereal.py +449 -0
  27. rslearn/data_sources/worldcover.py +144 -0
  28. rslearn/data_sources/worldpop.py +153 -0
  29. rslearn/data_sources/xyz_tiles.py +150 -107
  30. rslearn/dataset/__init__.py +8 -2
  31. rslearn/dataset/add_windows.py +2 -2
  32. rslearn/dataset/dataset.py +40 -51
  33. rslearn/dataset/handler_summaries.py +131 -0
  34. rslearn/dataset/manage.py +313 -74
  35. rslearn/dataset/materialize.py +431 -107
  36. rslearn/dataset/remap.py +29 -4
  37. rslearn/dataset/storage/__init__.py +1 -0
  38. rslearn/dataset/storage/file.py +202 -0
  39. rslearn/dataset/storage/storage.py +140 -0
  40. rslearn/dataset/window.py +181 -44
  41. rslearn/lightning_cli.py +454 -0
  42. rslearn/log_utils.py +24 -0
  43. rslearn/main.py +384 -181
  44. rslearn/models/anysat.py +215 -0
  45. rslearn/models/attention_pooling.py +177 -0
  46. rslearn/models/clay/clay.py +231 -0
  47. rslearn/models/clay/configs/metadata.yaml +295 -0
  48. rslearn/models/clip.py +68 -0
  49. rslearn/models/component.py +111 -0
  50. rslearn/models/concatenate_features.py +103 -0
  51. rslearn/models/conv.py +63 -0
  52. rslearn/models/croma.py +306 -0
  53. rslearn/models/detr/__init__.py +5 -0
  54. rslearn/models/detr/box_ops.py +103 -0
  55. rslearn/models/detr/detr.py +504 -0
  56. rslearn/models/detr/matcher.py +107 -0
  57. rslearn/models/detr/position_encoding.py +114 -0
  58. rslearn/models/detr/transformer.py +429 -0
  59. rslearn/models/detr/util.py +24 -0
  60. rslearn/models/dinov3.py +177 -0
  61. rslearn/models/faster_rcnn.py +30 -28
  62. rslearn/models/feature_center_crop.py +53 -0
  63. rslearn/models/fpn.py +19 -8
  64. rslearn/models/galileo/__init__.py +5 -0
  65. rslearn/models/galileo/galileo.py +595 -0
  66. rslearn/models/galileo/single_file_galileo.py +1678 -0
  67. rslearn/models/module_wrapper.py +65 -0
  68. rslearn/models/molmo.py +69 -0
  69. rslearn/models/multitask.py +384 -28
  70. rslearn/models/olmoearth_pretrain/__init__.py +1 -0
  71. rslearn/models/olmoearth_pretrain/model.py +421 -0
  72. rslearn/models/olmoearth_pretrain/norm.py +86 -0
  73. rslearn/models/panopticon.py +170 -0
  74. rslearn/models/panopticon_data/sensors/drone.yaml +32 -0
  75. rslearn/models/panopticon_data/sensors/enmap.yaml +904 -0
  76. rslearn/models/panopticon_data/sensors/goes.yaml +9 -0
  77. rslearn/models/panopticon_data/sensors/himawari.yaml +9 -0
  78. rslearn/models/panopticon_data/sensors/intuition.yaml +606 -0
  79. rslearn/models/panopticon_data/sensors/landsat8.yaml +84 -0
  80. rslearn/models/panopticon_data/sensors/modis_terra.yaml +99 -0
  81. rslearn/models/panopticon_data/sensors/qb2_ge1.yaml +34 -0
  82. rslearn/models/panopticon_data/sensors/sentinel1.yaml +85 -0
  83. rslearn/models/panopticon_data/sensors/sentinel2.yaml +97 -0
  84. rslearn/models/panopticon_data/sensors/superdove.yaml +60 -0
  85. rslearn/models/panopticon_data/sensors/wv23.yaml +63 -0
  86. rslearn/models/pick_features.py +17 -10
  87. rslearn/models/pooling_decoder.py +60 -7
  88. rslearn/models/presto/__init__.py +5 -0
  89. rslearn/models/presto/presto.py +297 -0
  90. rslearn/models/presto/single_file_presto.py +926 -0
  91. rslearn/models/prithvi.py +1147 -0
  92. rslearn/models/resize_features.py +59 -0
  93. rslearn/models/sam2_enc.py +13 -9
  94. rslearn/models/satlaspretrain.py +38 -18
  95. rslearn/models/simple_time_series.py +188 -77
  96. rslearn/models/singletask.py +24 -13
  97. rslearn/models/ssl4eo_s12.py +40 -30
  98. rslearn/models/swin.py +44 -32
  99. rslearn/models/task_embedding.py +250 -0
  100. rslearn/models/terramind.py +256 -0
  101. rslearn/models/trunk.py +139 -0
  102. rslearn/models/unet.py +68 -22
  103. rslearn/models/upsample.py +48 -0
  104. rslearn/models/use_croma.py +508 -0
  105. rslearn/template_params.py +26 -0
  106. rslearn/tile_stores/__init__.py +41 -18
  107. rslearn/tile_stores/default.py +409 -0
  108. rslearn/tile_stores/tile_store.py +236 -132
  109. rslearn/train/all_patches_dataset.py +530 -0
  110. rslearn/train/callbacks/adapters.py +53 -0
  111. rslearn/train/callbacks/freeze_unfreeze.py +348 -17
  112. rslearn/train/callbacks/gradients.py +129 -0
  113. rslearn/train/callbacks/peft.py +116 -0
  114. rslearn/train/data_module.py +444 -20
  115. rslearn/train/dataset.py +588 -235
  116. rslearn/train/lightning_module.py +192 -62
  117. rslearn/train/model_context.py +88 -0
  118. rslearn/train/optimizer.py +31 -0
  119. rslearn/train/prediction_writer.py +319 -84
  120. rslearn/train/scheduler.py +92 -0
  121. rslearn/train/tasks/classification.py +55 -28
  122. rslearn/train/tasks/detection.py +132 -76
  123. rslearn/train/tasks/embedding.py +120 -0
  124. rslearn/train/tasks/multi_task.py +28 -14
  125. rslearn/train/tasks/per_pixel_regression.py +291 -0
  126. rslearn/train/tasks/regression.py +161 -44
  127. rslearn/train/tasks/segmentation.py +428 -53
  128. rslearn/train/tasks/task.py +6 -5
  129. rslearn/train/transforms/__init__.py +1 -1
  130. rslearn/train/transforms/concatenate.py +54 -10
  131. rslearn/train/transforms/crop.py +29 -11
  132. rslearn/train/transforms/flip.py +18 -6
  133. rslearn/train/transforms/mask.py +78 -0
  134. rslearn/train/transforms/normalize.py +101 -17
  135. rslearn/train/transforms/pad.py +19 -7
  136. rslearn/train/transforms/resize.py +83 -0
  137. rslearn/train/transforms/select_bands.py +76 -0
  138. rslearn/train/transforms/sentinel1.py +75 -0
  139. rslearn/train/transforms/transform.py +89 -70
  140. rslearn/utils/__init__.py +2 -6
  141. rslearn/utils/array.py +8 -6
  142. rslearn/utils/feature.py +2 -2
  143. rslearn/utils/fsspec.py +90 -1
  144. rslearn/utils/geometry.py +347 -7
  145. rslearn/utils/get_utm_ups_crs.py +2 -3
  146. rslearn/utils/grid_index.py +5 -5
  147. rslearn/utils/jsonargparse.py +178 -0
  148. rslearn/utils/mp.py +4 -3
  149. rslearn/utils/raster_format.py +268 -116
  150. rslearn/utils/rtree_index.py +64 -17
  151. rslearn/utils/sqlite_index.py +7 -1
  152. rslearn/utils/vector_format.py +252 -97
  153. {rslearn-0.0.1.dist-info → rslearn-0.0.21.dist-info}/METADATA +532 -283
  154. rslearn-0.0.21.dist-info/RECORD +167 -0
  155. {rslearn-0.0.1.dist-info → rslearn-0.0.21.dist-info}/WHEEL +1 -1
  156. rslearn-0.0.21.dist-info/licenses/NOTICE +115 -0
  157. rslearn/data_sources/raster_source.py +0 -309
  158. rslearn/models/registry.py +0 -5
  159. rslearn/tile_stores/file.py +0 -242
  160. rslearn/utils/mgrs.py +0 -24
  161. rslearn/utils/utils.py +0 -22
  162. rslearn-0.0.1.dist-info/RECORD +0 -88
  163. /rslearn/{data_sources/geotiff.py → py.typed} +0 -0
  164. {rslearn-0.0.1.dist-info → rslearn-0.0.21.dist-info}/entry_points.txt +0 -0
  165. {rslearn-0.0.1.dist-info → rslearn-0.0.21.dist-info/licenses}/LICENSE +0 -0
  166. {rslearn-0.0.1.dist-info → rslearn-0.0.21.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,798 @@
1
+ """Data on Planetary Computer."""
2
+
3
+ import json
4
+ import os
5
+ import tempfile
6
+ import xml.etree.ElementTree as ET
7
+ from datetime import timedelta
8
+ from typing import Any
9
+
10
+ import affine
11
+ import numpy.typing as npt
12
+ import planetary_computer
13
+ import pystac
14
+ import pystac_client
15
+ import rasterio
16
+ import requests
17
+ import shapely
18
+ from rasterio.enums import Resampling
19
+ from upath import UPath
20
+
21
+ from rslearn.config import LayerConfig, QueryConfig
22
+ from rslearn.const import WGS84_PROJECTION
23
+ from rslearn.data_sources import DataSource, DataSourceContext, Item
24
+ from rslearn.data_sources.utils import match_candidate_items_to_window
25
+ from rslearn.dataset import Window
26
+ from rslearn.dataset.materialize import RasterMaterializer
27
+ from rslearn.log_utils import get_logger
28
+ from rslearn.tile_stores import TileStore, TileStoreWithLayer
29
+ from rslearn.utils.fsspec import join_upath
30
+ from rslearn.utils.geometry import PixelBounds, Projection, STGeometry
31
+ from rslearn.utils.raster_format import get_raster_projection_and_bounds
32
+
33
+ from .copernicus import get_harmonize_callback
34
+
35
+ logger = get_logger(__name__)
36
+
37
+
38
+ class PlanetaryComputerItem(Item):
39
+ """An item in the PlanetaryComputer data source."""
40
+
41
+ def __init__(self, name: str, geometry: STGeometry, asset_urls: dict[str, str]):
42
+ """Creates a new PlanetaryComputerItem.
43
+
44
+ Args:
45
+ name: unique name of the item
46
+ geometry: the spatial and temporal extent of the item
47
+ asset_urls: map from asset key to the unsigned asset URL.
48
+ """
49
+ super().__init__(name, geometry)
50
+ self.asset_urls = asset_urls
51
+
52
+ def serialize(self) -> dict[str, Any]:
53
+ """Serializes the item to a JSON-encodable dictionary."""
54
+ d = super().serialize()
55
+ d["asset_urls"] = self.asset_urls
56
+ return d
57
+
58
+ @staticmethod
59
+ def deserialize(d: dict[str, Any]) -> "PlanetaryComputerItem":
60
+ """Deserializes an item from a JSON-decoded dictionary."""
61
+ item = super(PlanetaryComputerItem, PlanetaryComputerItem).deserialize(d)
62
+ return PlanetaryComputerItem(
63
+ name=item.name,
64
+ geometry=item.geometry,
65
+ asset_urls=d["asset_urls"],
66
+ )
67
+
68
+
69
+ class PlanetaryComputer(DataSource, TileStore):
70
+ """Modality-agnostic data source for data on Microsoft Planetary Computer.
71
+
72
+ If there is a subclass available for a modality, it is recommended to use the
73
+ subclass since it provides additional functionality.
74
+
75
+ Otherwise, PlanetaryComputer can be configured with the collection name and a
76
+ dictionary of assets and bands to ingest.
77
+
78
+ See https://planetarycomputer.microsoft.com/ for details.
79
+
80
+ The PC_SDK_SUBSCRIPTION_KEY environment variable can be set for higher rate limits
81
+ but is not needed.
82
+ """
83
+
84
+ STAC_ENDPOINT = "https://planetarycomputer.microsoft.com/api/stac/v1"
85
+
86
+ # Default threshold for recreating the STAC client to prevent memory leaks
87
+ # from the pystac Catalog's resolved objects cache growing unbounded
88
+ DEFAULT_MAX_ITEMS_PER_CLIENT = 1000
89
+
90
+ def __init__(
91
+ self,
92
+ collection_name: str,
93
+ asset_bands: dict[str, list[str]],
94
+ query: dict[str, Any] | None = None,
95
+ sort_by: str | None = None,
96
+ sort_ascending: bool = True,
97
+ timeout: timedelta = timedelta(seconds=10),
98
+ skip_items_missing_assets: bool = False,
99
+ cache_dir: str | None = None,
100
+ max_items_per_client: int | None = None,
101
+ context: DataSourceContext = DataSourceContext(),
102
+ ):
103
+ """Initialize a new PlanetaryComputer instance.
104
+
105
+ Args:
106
+ collection_name: the STAC collection name on Planetary Computer.
107
+ asset_bands: assets to ingest, mapping from asset name to the list of bands
108
+ in that asset.
109
+ query: optional query argument to STAC searches.
110
+ sort_by: sort by this property in the STAC items.
111
+ sort_ascending: whether to sort ascending (or descending).
112
+ timeout: timeout for API requests.
113
+ skip_items_missing_assets: skip STAC items that are missing any of the
114
+ assets in asset_bands during get_items.
115
+ cache_dir: optional directory to cache items by name, including asset URLs.
116
+ If not set, there will be no cache and instead STAC requests will be
117
+ needed each time.
118
+ max_items_per_client: number of STAC items to process before recreating
119
+ the client to prevent memory leaks from the resolved objects cache.
120
+ Defaults to DEFAULT_MAX_ITEMS_PER_CLIENT.
121
+ context: the data source context.
122
+ """
123
+ self.collection_name = collection_name
124
+ self.asset_bands = asset_bands
125
+ self.query = query
126
+ self.sort_by = sort_by
127
+ self.sort_ascending = sort_ascending
128
+ self.timeout = timeout
129
+ self.skip_items_missing_assets = skip_items_missing_assets
130
+ self.max_items_per_client = (
131
+ max_items_per_client or self.DEFAULT_MAX_ITEMS_PER_CLIENT
132
+ )
133
+
134
+ if cache_dir is not None:
135
+ if context.ds_path is not None:
136
+ self.cache_dir = join_upath(context.ds_path, cache_dir)
137
+ else:
138
+ self.cache_dir = UPath(cache_dir)
139
+
140
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
141
+ else:
142
+ self.cache_dir = None
143
+
144
+ self.client: pystac_client.Client | None = None
145
+ self._client_item_count = 0
146
+
147
+ def _load_client(
148
+ self,
149
+ ) -> pystac_client.Client:
150
+ """Lazily load pystac client.
151
+
152
+ We don't load it when creating the data source because it takes time and caller
153
+ may not be calling get_items. Additionally, loading it during the get_items
154
+ call enables leveraging the retry loop functionality in
155
+ prepare_dataset_windows.
156
+
157
+ Note: We periodically recreate the client to prevent memory leaks from the
158
+ pystac Catalog's resolved objects cache, which grows unbounded as STAC items
159
+ are deserialized and cached. The cache cannot be cleared or disabled.
160
+ """
161
+ if self.client is None:
162
+ logger.info("Creating initial STAC client")
163
+ self.client = pystac_client.Client.open(self.STAC_ENDPOINT)
164
+ return self.client
165
+
166
+ if self._client_item_count < self.max_items_per_client:
167
+ return self.client
168
+
169
+ # Recreate client to clear the resolved objects cache
170
+ current_client = self.client
171
+ logger.debug(
172
+ "Recreating STAC client after processing %d items (threshold: %d)",
173
+ self._client_item_count,
174
+ self.max_items_per_client,
175
+ )
176
+ client_root = current_client.get_root()
177
+ client_root.clear_links()
178
+ client_root.clear_items()
179
+ client_root.clear_children()
180
+ self._client_item_count = 0
181
+ self.client = pystac_client.Client.open(self.STAC_ENDPOINT)
182
+ return self.client
183
+
184
+ def _stac_item_to_item(self, stac_item: pystac.Item) -> PlanetaryComputerItem:
185
+ shp = shapely.geometry.shape(stac_item.geometry)
186
+
187
+ # Get time range.
188
+ metadata = stac_item.common_metadata
189
+ if metadata.start_datetime is not None and metadata.end_datetime is not None:
190
+ time_range = (
191
+ metadata.start_datetime,
192
+ metadata.end_datetime,
193
+ )
194
+ elif stac_item.datetime is not None:
195
+ time_range = (stac_item.datetime, stac_item.datetime)
196
+ else:
197
+ raise ValueError(
198
+ f"item {stac_item.id} unexpectedly missing start_datetime, end_datetime, and datetime"
199
+ )
200
+
201
+ geom = STGeometry(WGS84_PROJECTION, shp, time_range)
202
+ asset_urls = {
203
+ asset_key: asset_obj.href
204
+ for asset_key, asset_obj in stac_item.assets.items()
205
+ }
206
+ return PlanetaryComputerItem(stac_item.id, geom, asset_urls)
207
+
208
+ def get_item_by_name(self, name: str) -> PlanetaryComputerItem:
209
+ """Gets an item by name.
210
+
211
+ Args:
212
+ name: the name of the item to get
213
+
214
+ Returns:
215
+ the item object
216
+ """
217
+ # If cache_dir is set, we cache the item. First here we check if it is already
218
+ # in the cache.
219
+ cache_fname: UPath | None = None
220
+ if self.cache_dir:
221
+ cache_fname = self.cache_dir / f"{name}.json"
222
+ if cache_fname is not None and cache_fname.exists():
223
+ with cache_fname.open() as f:
224
+ return PlanetaryComputerItem.deserialize(json.load(f))
225
+
226
+ # No cache or not in cache, so we need to make the STAC request.
227
+ logger.debug("Getting STAC item {name}")
228
+ client = self._load_client()
229
+
230
+ search_result = client.search(ids=[name], collections=[self.collection_name])
231
+ stac_items = list(search_result.items())
232
+
233
+ if not stac_items:
234
+ raise ValueError(
235
+ f"Item {name} not found in collection {self.collection_name}"
236
+ )
237
+ if len(stac_items) > 1:
238
+ raise ValueError(
239
+ f"Multiple items found for ID {name} in collection {self.collection_name}"
240
+ )
241
+
242
+ stac_item = stac_items[0]
243
+ item = self._stac_item_to_item(stac_item)
244
+
245
+ # Track items processed for client recreation threshold (after deserialization)
246
+ self._client_item_count += 1
247
+
248
+ # Finally we cache it if cache_dir is set.
249
+ if cache_fname is not None:
250
+ with cache_fname.open("w") as f:
251
+ json.dump(item.serialize(), f)
252
+
253
+ return item
254
+
255
+ def get_items(
256
+ self, geometries: list[STGeometry], query_config: QueryConfig
257
+ ) -> list[list[list[PlanetaryComputerItem]]]:
258
+ """Get a list of items in the data source intersecting the given geometries.
259
+
260
+ Args:
261
+ geometries: the spatiotemporal geometries
262
+ query_config: the query configuration
263
+
264
+ Returns:
265
+ List of groups of items that should be retrieved for each geometry.
266
+ """
267
+ client = self._load_client()
268
+
269
+ groups = []
270
+ for geometry in geometries:
271
+ # Get potentially relevant items from the collection by performing one search
272
+ # for each requested geometry.
273
+ wgs84_geometry = geometry.to_projection(WGS84_PROJECTION)
274
+ logger.debug("performing STAC search for geometry %s", wgs84_geometry)
275
+ result = client.search(
276
+ collections=[self.collection_name],
277
+ intersects=shapely.to_geojson(wgs84_geometry.shp),
278
+ datetime=wgs84_geometry.time_range,
279
+ query=self.query,
280
+ )
281
+ stac_items = [item for item in result.items()]
282
+ # Track items processed for client recreation threshold (after deserialization)
283
+ self._client_item_count += len(stac_items)
284
+ logger.debug("STAC search yielded %d items", len(stac_items))
285
+
286
+ if self.skip_items_missing_assets:
287
+ # Filter out items that are missing any of the assets in self.asset_bands.
288
+ good_stac_items = []
289
+ for stac_item in stac_items:
290
+ good = True
291
+ for asset_key in self.asset_bands.keys():
292
+ if asset_key in stac_item.assets:
293
+ continue
294
+ good = False
295
+ break
296
+ if good:
297
+ good_stac_items.append(stac_item)
298
+ logger.debug(
299
+ "skip_items_missing_assets filter from %d to %d items",
300
+ len(stac_items),
301
+ len(good_stac_items),
302
+ )
303
+ stac_items = good_stac_items
304
+
305
+ if self.sort_by is not None:
306
+ stac_items.sort(
307
+ key=lambda stac_item: stac_item.properties[self.sort_by],
308
+ reverse=not self.sort_ascending,
309
+ )
310
+
311
+ candidate_items = [
312
+ self._stac_item_to_item(stac_item) for stac_item in stac_items
313
+ ]
314
+
315
+ # Since we made the STAC request, might as well save these to the cache.
316
+ if self.cache_dir is not None:
317
+ for item in candidate_items:
318
+ cache_fname = self.cache_dir / f"{item.name}.json"
319
+ if cache_fname.exists():
320
+ continue
321
+ with cache_fname.open("w") as f:
322
+ json.dump(item.serialize(), f)
323
+
324
+ cur_groups = match_candidate_items_to_window(
325
+ geometry, candidate_items, query_config
326
+ )
327
+ groups.append(cur_groups)
328
+
329
+ return groups
330
+
331
+ def deserialize_item(self, serialized_item: Any) -> PlanetaryComputerItem:
332
+ """Deserializes an item from JSON-decoded data."""
333
+ assert isinstance(serialized_item, dict)
334
+ return PlanetaryComputerItem.deserialize(serialized_item)
335
+
336
+ def ingest(
337
+ self,
338
+ tile_store: TileStoreWithLayer,
339
+ items: list[PlanetaryComputerItem],
340
+ geometries: list[list[STGeometry]],
341
+ ) -> None:
342
+ """Ingest items into the given tile store.
343
+
344
+ Args:
345
+ tile_store: the tile store to ingest into
346
+ items: the items to ingest
347
+ geometries: a list of geometries needed for each item
348
+ """
349
+ for item in items:
350
+ for asset_key, band_names in self.asset_bands.items():
351
+ if asset_key not in item.asset_urls:
352
+ continue
353
+ if tile_store.is_raster_ready(item.name, band_names):
354
+ continue
355
+
356
+ asset_url = planetary_computer.sign(item.asset_urls[asset_key])
357
+
358
+ with tempfile.TemporaryDirectory() as tmp_dir:
359
+ local_fname = os.path.join(tmp_dir, f"{asset_key}.tif")
360
+ logger.debug(
361
+ "PlanetaryComputer download item %s asset %s to %s",
362
+ item.name,
363
+ asset_key,
364
+ local_fname,
365
+ )
366
+ with requests.get(
367
+ asset_url, stream=True, timeout=self.timeout.total_seconds()
368
+ ) as r:
369
+ r.raise_for_status()
370
+ with open(local_fname, "wb") as f:
371
+ for chunk in r.iter_content(chunk_size=8192):
372
+ f.write(chunk)
373
+
374
+ logger.debug(
375
+ "PlanetaryComputer ingest item %s asset %s",
376
+ item.name,
377
+ asset_key,
378
+ )
379
+ tile_store.write_raster_file(
380
+ item.name, band_names, UPath(local_fname)
381
+ )
382
+
383
+ logger.debug(
384
+ "PlanetaryComputer done ingesting item %s asset %s",
385
+ item.name,
386
+ asset_key,
387
+ )
388
+
389
+ def is_raster_ready(
390
+ self, layer_name: str, item_name: str, bands: list[str]
391
+ ) -> bool:
392
+ """Checks if this raster has been written to the store.
393
+
394
+ Args:
395
+ layer_name: the layer name or alias.
396
+ item_name: the item.
397
+ bands: the list of bands identifying which specific raster to read.
398
+
399
+ Returns:
400
+ whether there is a raster in the store matching the source, item, and
401
+ bands.
402
+ """
403
+ # Always ready since we wrap accesses to Planetary Computer.
404
+ return True
405
+
406
+ def get_raster_bands(self, layer_name: str, item_name: str) -> list[list[str]]:
407
+ """Get the sets of bands that have been stored for the specified item.
408
+
409
+ Args:
410
+ layer_name: the layer name or alias.
411
+ item_name: the item.
412
+
413
+ Returns:
414
+ a list of lists of bands that are in the tile store (with one raster
415
+ stored corresponding to each inner list). If no rasters are ready for
416
+ this item, returns empty list.
417
+ """
418
+ if self.skip_items_missing_assets:
419
+ # In this case we can assume that the item has all of the assets.
420
+ return list(self.asset_bands.values())
421
+
422
+ # Otherwise we have to lookup the STAC item to see which assets it has.
423
+ # Here we use get_item_by_name since it handles caching.
424
+ item = self.get_item_by_name(item_name)
425
+ all_bands = []
426
+ for asset_key, band_names in self.asset_bands.items():
427
+ if asset_key not in item.asset_urls:
428
+ continue
429
+ all_bands.append(band_names)
430
+ return all_bands
431
+
432
+ def _get_asset_by_band(self, bands: list[str]) -> str:
433
+ """Get the name of the asset based on the band names."""
434
+ for asset_key, asset_bands in self.asset_bands.items():
435
+ if bands == asset_bands:
436
+ return asset_key
437
+
438
+ raise ValueError(f"no raster with bands {bands}")
439
+
440
+ def get_raster_bounds(
441
+ self, layer_name: str, item_name: str, bands: list[str], projection: Projection
442
+ ) -> PixelBounds:
443
+ """Get the bounds of the raster in the specified projection.
444
+
445
+ Args:
446
+ layer_name: the layer name or alias.
447
+ item_name: the item to check.
448
+ bands: the list of bands identifying which specific raster to read. These
449
+ bands must match the bands of a stored raster.
450
+ projection: the projection to get the raster's bounds in.
451
+
452
+ Returns:
453
+ the bounds of the raster in the projection.
454
+ """
455
+ item = self.get_item_by_name(item_name)
456
+ geom = item.geometry.to_projection(projection)
457
+ return (
458
+ int(geom.shp.bounds[0]),
459
+ int(geom.shp.bounds[1]),
460
+ int(geom.shp.bounds[2]),
461
+ int(geom.shp.bounds[3]),
462
+ )
463
+
464
+ def read_raster(
465
+ self,
466
+ layer_name: str,
467
+ item_name: str,
468
+ bands: list[str],
469
+ projection: Projection,
470
+ bounds: PixelBounds,
471
+ resampling: Resampling = Resampling.bilinear,
472
+ ) -> npt.NDArray[Any]:
473
+ """Read raster data from the store.
474
+
475
+ Args:
476
+ layer_name: the layer name or alias.
477
+ item_name: the item to read.
478
+ bands: the list of bands identifying which specific raster to read. These
479
+ bands must match the bands of a stored raster.
480
+ projection: the projection to read in.
481
+ bounds: the bounds to read.
482
+ resampling: the resampling method to use in case reprojection is needed.
483
+
484
+ Returns:
485
+ the raster data
486
+ """
487
+ asset_key = self._get_asset_by_band(bands)
488
+ item = self.get_item_by_name(item_name)
489
+ asset_url = planetary_computer.sign(item.asset_urls[asset_key])
490
+
491
+ # Construct the transform to use for the warped dataset.
492
+ wanted_transform = affine.Affine(
493
+ projection.x_resolution,
494
+ 0,
495
+ bounds[0] * projection.x_resolution,
496
+ 0,
497
+ projection.y_resolution,
498
+ bounds[1] * projection.y_resolution,
499
+ )
500
+
501
+ with rasterio.open(asset_url) as src:
502
+ with rasterio.vrt.WarpedVRT(
503
+ src,
504
+ crs=projection.crs,
505
+ transform=wanted_transform,
506
+ width=bounds[2] - bounds[0],
507
+ height=bounds[3] - bounds[1],
508
+ resampling=resampling,
509
+ ) as vrt:
510
+ return vrt.read()
511
+
512
+ def materialize(
513
+ self,
514
+ window: Window,
515
+ item_groups: list[list[Item]],
516
+ layer_name: str,
517
+ layer_cfg: LayerConfig,
518
+ ) -> None:
519
+ """Materialize data for the window.
520
+
521
+ Args:
522
+ window: the window to materialize
523
+ item_groups: the items from get_items
524
+ layer_name: the name of this layer
525
+ layer_cfg: the config of this layer
526
+ """
527
+ RasterMaterializer().materialize(
528
+ TileStoreWithLayer(self, layer_name),
529
+ window,
530
+ layer_name,
531
+ layer_cfg,
532
+ item_groups,
533
+ )
534
+
535
+
536
+ class Sentinel2(PlanetaryComputer):
537
+ """A data source for Sentinel-2 L2A data on Microsoft Planetary Computer.
538
+
539
+ See https://planetarycomputer.microsoft.com/dataset/sentinel-2-l2a.
540
+ """
541
+
542
+ COLLECTION_NAME = "sentinel-2-l2a"
543
+
544
+ BANDS = {
545
+ "B01": ["B01"],
546
+ "B02": ["B02"],
547
+ "B03": ["B03"],
548
+ "B04": ["B04"],
549
+ "B05": ["B05"],
550
+ "B06": ["B06"],
551
+ "B07": ["B07"],
552
+ "B08": ["B08"],
553
+ "B09": ["B09"],
554
+ "B11": ["B11"],
555
+ "B12": ["B12"],
556
+ "B8A": ["B8A"],
557
+ "visual": ["R", "G", "B"],
558
+ }
559
+
560
+ def __init__(
561
+ self,
562
+ harmonize: bool = False,
563
+ assets: list[str] | None = None,
564
+ context: DataSourceContext = DataSourceContext(),
565
+ **kwargs: Any,
566
+ ):
567
+ """Initialize a new Sentinel2 instance.
568
+
569
+ Args:
570
+ harmonize: harmonize pixel values across different processing baselines,
571
+ see https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_S2_SR_HARMONIZED
572
+ assets: list of asset names to ingest, or None to ingest all assets. This
573
+ is only used if the layer config is missing from the context.
574
+ context: the data source context.
575
+ kwargs: other arguments to pass to PlanetaryComputer.
576
+ """
577
+ self.harmonize = harmonize
578
+
579
+ # Determine which assets we need based on the bands in the layer config.
580
+ if context.layer_config is not None:
581
+ asset_bands: dict[str, list[str]] = {}
582
+ for asset_key, band_names in self.BANDS.items():
583
+ # See if the bands provided by this asset intersect with the bands in
584
+ # at least one configured band set.
585
+ for band_set in context.layer_config.band_sets:
586
+ if not set(band_set.bands).intersection(set(band_names)):
587
+ continue
588
+ asset_bands[asset_key] = band_names
589
+ break
590
+ elif assets is not None:
591
+ asset_bands = {asset_key: self.BANDS[asset_key] for asset_key in assets}
592
+ else:
593
+ asset_bands = self.BANDS
594
+
595
+ super().__init__(
596
+ collection_name=self.COLLECTION_NAME,
597
+ asset_bands=asset_bands,
598
+ # Skip since all of the items should have the same assets.
599
+ skip_items_missing_assets=True,
600
+ context=context,
601
+ **kwargs,
602
+ )
603
+
604
+ def _get_product_xml(self, item: PlanetaryComputerItem) -> ET.Element:
605
+ asset_url = planetary_computer.sign(item.asset_urls["product-metadata"])
606
+ response = requests.get(asset_url, timeout=self.timeout.total_seconds())
607
+ response.raise_for_status()
608
+ return ET.fromstring(response.content)
609
+
610
+ def ingest(
611
+ self,
612
+ tile_store: TileStoreWithLayer,
613
+ items: list[PlanetaryComputerItem],
614
+ geometries: list[list[STGeometry]],
615
+ ) -> None:
616
+ """Ingest items into the given tile store.
617
+
618
+ Args:
619
+ tile_store: the tile store to ingest into
620
+ items: the items to ingest
621
+ geometries: a list of geometries needed for each item
622
+ """
623
+ for item in items:
624
+ for asset_key, band_names in self.asset_bands.items():
625
+ if tile_store.is_raster_ready(item.name, band_names):
626
+ continue
627
+
628
+ asset_url = planetary_computer.sign(item.asset_urls[asset_key])
629
+
630
+ with tempfile.TemporaryDirectory() as tmp_dir:
631
+ local_fname = os.path.join(tmp_dir, f"{asset_key}.tif")
632
+ logger.debug(
633
+ "PlanetaryComputer download item %s asset %s to %s",
634
+ item.name,
635
+ asset_key,
636
+ local_fname,
637
+ )
638
+ with requests.get(
639
+ asset_url, stream=True, timeout=self.timeout.total_seconds()
640
+ ) as r:
641
+ r.raise_for_status()
642
+ with open(local_fname, "wb") as f:
643
+ for chunk in r.iter_content(chunk_size=8192):
644
+ f.write(chunk)
645
+
646
+ logger.debug(
647
+ "PlanetaryComputer ingest item %s asset %s",
648
+ item.name,
649
+ asset_key,
650
+ )
651
+
652
+ # Harmonize values if needed.
653
+ # TCI does not need harmonization.
654
+ harmonize_callback = None
655
+ if self.harmonize and asset_key != "visual":
656
+ harmonize_callback = get_harmonize_callback(
657
+ self._get_product_xml(item)
658
+ )
659
+
660
+ if harmonize_callback is not None:
661
+ # In this case we need to read the array, convert the pixel
662
+ # values, and pass modified array directly to the TileStore.
663
+ with rasterio.open(local_fname) as src:
664
+ array = src.read()
665
+ projection, bounds = get_raster_projection_and_bounds(src)
666
+ array = harmonize_callback(array)
667
+ tile_store.write_raster(
668
+ item.name, band_names, projection, bounds, array
669
+ )
670
+
671
+ else:
672
+ tile_store.write_raster_file(
673
+ item.name, band_names, UPath(local_fname)
674
+ )
675
+
676
+ logger.debug(
677
+ "PlanetaryComputer done ingesting item %s asset %s",
678
+ item.name,
679
+ asset_key,
680
+ )
681
+
682
+ def read_raster(
683
+ self,
684
+ layer_name: str,
685
+ item_name: str,
686
+ bands: list[str],
687
+ projection: Projection,
688
+ bounds: PixelBounds,
689
+ resampling: Resampling = Resampling.bilinear,
690
+ ) -> npt.NDArray[Any]:
691
+ """Read raster data from the store.
692
+
693
+ Args:
694
+ layer_name: the layer name or alias.
695
+ item_name: the item to read.
696
+ bands: the list of bands identifying which specific raster to read. These
697
+ bands must match the bands of a stored raster.
698
+ projection: the projection to read in.
699
+ bounds: the bounds to read.
700
+ resampling: the resampling method to use in case reprojection is needed.
701
+
702
+ Returns:
703
+ the raster data
704
+ """
705
+ # We override read_raster because we may need to harmonize the data.
706
+ raw_data = super().read_raster(
707
+ layer_name, item_name, bands, projection, bounds, resampling=resampling
708
+ )
709
+
710
+ # TCI (visual) image does not need harmonization.
711
+ if not self.harmonize or bands == self.BANDS["visual"]:
712
+ return raw_data
713
+
714
+ item = self.get_item_by_name(item_name)
715
+ harmonize_callback = get_harmonize_callback(self._get_product_xml(item))
716
+
717
+ if harmonize_callback is None:
718
+ return raw_data
719
+
720
+ array = harmonize_callback(raw_data)
721
+ return array
722
+
723
+
724
+ class Sentinel1(PlanetaryComputer):
725
+ """A data source for Sentinel-1 data on Microsoft Planetary Computer.
726
+
727
+ This uses the radiometrically corrected data.
728
+
729
+ See https://planetarycomputer.microsoft.com/dataset/sentinel-1-rtc.
730
+ """
731
+
732
+ COLLECTION_NAME = "sentinel-1-rtc"
733
+
734
+ def __init__(
735
+ self,
736
+ band_names: list[str] | None = None,
737
+ context: DataSourceContext = DataSourceContext(),
738
+ **kwargs: Any,
739
+ ):
740
+ """Initialize a new Sentinel1 instance.
741
+
742
+ Args:
743
+ band_names: list of bands to try to ingest, if the layer config is missing
744
+ from the context.
745
+ context: the data source context.
746
+ kwargs: additional arguments to pass to PlanetaryComputer.
747
+ """
748
+ # Get band names from the config if possible. If it isn't in the context, then
749
+ # we have to use the provided band names.
750
+ if context.layer_config is not None:
751
+ band_names = list(
752
+ {
753
+ band
754
+ for band_set in context.layer_config.band_sets
755
+ for band in band_set.bands
756
+ }
757
+ )
758
+ if band_names is None:
759
+ raise ValueError(
760
+ "band_names must be set if layer config is not in the context"
761
+ )
762
+ # For Sentinel-1, the asset key should be the same as the band name (and all
763
+ # assets have one band).
764
+ asset_bands = {band: [band] for band in band_names}
765
+ super().__init__(
766
+ collection_name=self.COLLECTION_NAME,
767
+ asset_bands=asset_bands,
768
+ context=context,
769
+ **kwargs,
770
+ )
771
+
772
+
773
+ class Naip(PlanetaryComputer):
774
+ """A data source for NAIP data on Microsoft Planetary Computer.
775
+
776
+ See https://planetarycomputer.microsoft.com/dataset/naip.
777
+ """
778
+
779
+ COLLECTION_NAME = "naip"
780
+ ASSET_BANDS = {"image": ["R", "G", "B", "NIR"]}
781
+
782
+ def __init__(
783
+ self,
784
+ context: DataSourceContext = DataSourceContext(),
785
+ **kwargs: Any,
786
+ ):
787
+ """Initialize a new Naip instance.
788
+
789
+ Args:
790
+ context: the data source context.
791
+ kwargs: additional arguments to pass to PlanetaryComputer.
792
+ """
793
+ super().__init__(
794
+ collection_name=self.COLLECTION_NAME,
795
+ asset_bands=self.ASSET_BANDS,
796
+ context=context,
797
+ **kwargs,
798
+ )