rslearn 0.0.1__py3-none-any.whl → 0.0.21__py3-none-any.whl

rslearn/data_sources/aws_open_data.py

@@ -2,10 +2,11 @@
 
 import io
 import json
+import os
 import tempfile
 import xml.etree.ElementTree as ET
 from collections.abc import Callable, Generator
-from datetime import datetime, timedelta, timezone
+from datetime import UTC, datetime
 from enum import Enum
 from typing import Any, BinaryIO
 
@@ -14,7 +15,6 @@ import dateutil.parser
 import fiona
 import fiona.transform
 import numpy.typing as npt
-import pytimeparse
 import rasterio
 import shapely
 import tqdm
@@ -22,27 +22,21 @@ from rasterio.crs import CRS
 from upath import UPath
 
 import rslearn.data_sources.utils
-import rslearn.utils.mgrs
-from rslearn.config import LayerConfig, RasterLayerConfig
 from rslearn.const import SHAPEFILE_AUX_EXTENSIONS, WGS84_EPSG, WGS84_PROJECTION
-from rslearn.tile_stores import PrefixedTileStore, TileStore
-from rslearn.utils import (
-    GridIndex,
-    Projection,
-    STGeometry,
-    daterange,
-)
+from rslearn.tile_stores import TileStoreWithLayer
+from rslearn.utils import GridIndex, Projection, STGeometry, daterange
 from rslearn.utils.fsspec import get_upath_local, join_upath, open_atomic
+from rslearn.utils.raster_format import get_raster_projection_and_bounds
 
-from .copernicus import get_harmonize_callback
+from .copernicus import get_harmonize_callback, get_sentinel2_tiles
 from .data_source import (
     DataSource,
+    DataSourceContext,
     Item,
     ItemLookupDataSource,
     QueryConfig,
     RetrieveItemDataSource,
 )
-from .raster_source import get_needed_projections, ingest_raster
 
 
 class NaipItem(Item):
@@ -66,7 +60,7 @@ class NaipItem(Item):
         return d
 
     @staticmethod
-    def deserialize(d: dict) -> Item:
+    def deserialize(d: dict) -> "NaipItem":
         """Deserializes an item from a JSON-decoded dictionary."""
         item = super(NaipItem, NaipItem).deserialize(d)
         return NaipItem(
@@ -89,16 +83,15 @@ class Naip(DataSource):
 
     def __init__(
         self,
-        config: LayerConfig,
-        index_cache_dir: UPath,
+        index_cache_dir: str,
         use_rtree_index: bool = False,
         states: list[str] | None = None,
         years: list[int] | None = None,
+        context: DataSourceContext = DataSourceContext(),
     ) -> None:
         """Initialize a new Naip instance.
 
         Args:
-            config: the LayerConfig of the layer containing this data source.
             index_cache_dir: directory to cache index shapefiles.
             use_rtree_index: whether to create an rtree index to enable faster lookups
                 (default false)
@@ -106,40 +99,30 @@ class Naip(DataSource):
                 the search. If use_rtree_index is enabled, the rtree will only be
                 populated with data from these states.
             years: optional list of years to restrict the search
+            context: the data source context.
         """
-        self.config = config
-        self.index_cache_dir = index_cache_dir
+        # If context is provided, we join the directory with the dataset path,
+        # otherwise we treat it directly as UPath.
+        if context.ds_path is not None:
+            self.index_cache_dir = join_upath(context.ds_path, index_cache_dir)
+        else:
+            self.index_cache_dir = UPath(index_cache_dir)
+
         self.states = states
         self.years = years
 
-        self.bucket = boto3.resource("s3").Bucket(self.bucket_name)
+        self.index_cache_dir.mkdir(parents=True, exist_ok=True)
 
+        self.bucket = boto3.resource("s3").Bucket(self.bucket_name)
+        self.rtree_index: Any | None = None
         if use_rtree_index:
             from rslearn.utils.rtree_index import RtreeIndex, get_cached_rtree
 
-            def build_fn(index: RtreeIndex):
+            def build_fn(index: RtreeIndex) -> None:
                 for item in self._read_index_shapefiles(desc="Building rtree index"):
                     index.insert(item.geometry.shp.bounds, json.dumps(item.serialize()))
 
-            self.rtree_tmp_dir = tempfile.TemporaryDirectory()
-            self.rtree_index = get_cached_rtree(
-                self.index_cache_dir, self.rtree_tmp_dir.name, build_fn
-            )
-        else:
-            self.rtree_index = None
-
-    @staticmethod
-    def from_config(config: LayerConfig, ds_path: UPath) -> "Naip":
-        """Creates a new Naip instance from a configuration dictionary."""
-        assert isinstance(config, RasterLayerConfig)
-        d = config.data_source.config_dict
-        kwargs = dict(
-            config=config,
-            index_cache_dir=join_upath(ds_path, d["index_cache_dir"]),
-        )
-        if "use_rtree_index" in d:
-            kwargs["use_rtree_index"] = d["use_rtree_index"]
-        return Naip(**kwargs)
+            self.rtree_index = get_cached_rtree(self.index_cache_dir, build_fn)
 
     def _download_manifest(self) -> UPath:
         """Download the manifest that enumerates files in the bucket.
@@ -149,7 +132,7 @@ class Naip(DataSource):
         """
         manifest_path = self.index_cache_dir / self.manifest_fname
         if not manifest_path.exists():
-            with manifest_path.open("wb") as dst:
+            with open_atomic(manifest_path, "wb") as dst:
                 self.bucket.download_fileobj(
                     self.manifest_fname,
                     dst,
@@ -195,7 +178,9 @@ class Naip(DataSource):
                     blob_path, dst, ExtraArgs={"RequestPayer": "requester"}
                 )
 
-    def _read_index_shapefiles(self, desc=None) -> Generator[NaipItem, None, None]:
+    def _read_index_shapefiles(
+        self, desc: str | None = None
+    ) -> Generator[NaipItem, None, None]:
         """Read the index shapefiles and yield NaipItems corresponding to each image."""
         self._download_index_shapefiles()
 
@@ -275,7 +260,7 @@ class Naip(DataSource):
                         else:
                             src_img_date = fname_parts[5]
                         time = datetime.strptime(src_img_date, "%Y%m%d").replace(
-                            tzinfo=timezone.utc
+                            tzinfo=UTC
                         )
 
                         geometry = STGeometry(WGS84_PROJECTION, shp, (time, time))
@@ -288,7 +273,7 @@ class Naip(DataSource):
 
     def get_items(
         self, geometries: list[STGeometry], query_config: QueryConfig
-    ) -> list[list[list[Item]]]:
+    ) -> list[list[list[NaipItem]]]:
         """Get a list of items in the data source intersecting the given geometries.
 
         Args:
@@ -302,7 +287,7 @@ class Naip(DataSource):
             geometry.to_projection(WGS84_PROJECTION) for geometry in geometries
         ]
 
-        items = [[] for _ in geometries]
+        items: list = [[] for _ in geometries]
         if self.rtree_index:
             for idx, geometry in enumerate(wgs84_geometries):
                 encoded_items = self.rtree_index.query(geometry.shp.bounds)
@@ -331,15 +316,15 @@ class Naip(DataSource):
             groups.append(cur_groups)
         return groups
 
-    def deserialize_item(self, serialized_item: Any) -> Item:
+    def deserialize_item(self, serialized_item: Any) -> NaipItem:
         """Deserializes an item from JSON-decoded data."""
         assert isinstance(serialized_item, dict)
         return NaipItem.deserialize(serialized_item)
 
     def ingest(
         self,
-        tile_store: TileStore,
-        items: list[Item],
+        tile_store: TileStoreWithLayer,
+        items: list[NaipItem],
         geometries: list[list[STGeometry]],
     ) -> None:
         """Ingest items into the given tile store.
@@ -349,29 +334,17 @@ class Naip(DataSource):
             items: the items to ingest
             geometries: a list of geometries needed for each item
         """
-        for item, cur_geometries in zip(items, geometries):
+        for item in items:
             bands = ["R", "G", "B", "IR"]
-            cur_tile_store = PrefixedTileStore(tile_store, (item.name, "_".join(bands)))
-            needed_projections = get_needed_projections(
-                cur_tile_store, bands, self.config.band_sets, cur_geometries
-            )
-            if not needed_projections:
+            if tile_store.is_raster_ready(item.name, bands):
                 continue
 
-            buf = io.BytesIO()
-            self.bucket.download_fileobj(
-                item.blob_path, buf, ExtraArgs={"RequestPayer": "requester"}
-            )
-            buf.seek(0)
-            with rasterio.open(buf) as raster:
-                for projection in needed_projections:
-                    ingest_raster(
-                        tile_store=cur_tile_store,
-                        raster=raster,
-                        projection=projection,
-                        time_range=item.geometry.time_range,
-                        layer_config=self.config,
-                    )
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                fname = os.path.join(tmp_dir, item.blob_path.split("/")[-1])
+                self.bucket.download_file(
+                    item.blob_path, fname, ExtraArgs={"RequestPayer": "requester"}
+                )
+                tile_store.write_raster_file(item.name, bands, UPath(fname))
 
 
 class Sentinel2Modality(Enum):
@@ -407,7 +380,7 @@ class Sentinel2Item(Item):
         return d
 
     @staticmethod
-    def deserialize(d: dict) -> Item:
+    def deserialize(d: dict) -> "Sentinel2Item":
         """Deserializes an item from a JSON-decoded dictionary."""
         if "name" not in d:
             d["name"] = d["blob_path"].split("/")[-1].split(".tif")[0]
@@ -420,7 +393,9 @@ class Sentinel2Item(Item):
         )
 
 
-class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
+class Sentinel2(
+    ItemLookupDataSource[Sentinel2Item], RetrieveItemDataSource[Sentinel2Item]
+):
     """A data source for Sentinel-2 L1C and L2A imagery on AWS.
 
     Specifically, uses the sentinel-s2-l1c and sentinel-s2-l2a S3 buckets maintained by
@@ -474,61 +449,39 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
 
     def __init__(
         self,
-        config: LayerConfig,
         modality: Sentinel2Modality,
-        metadata_cache_dir: UPath,
-        max_time_delta: timedelta = timedelta(days=30),
+        metadata_cache_dir: str,
         sort_by: str | None = None,
         harmonize: bool = False,
+        context: DataSourceContext = DataSourceContext(),
     ) -> None:
         """Initialize a new Sentinel2 instance.
 
         Args:
-            config: the LayerConfig of the layer containing this data source.
             modality: L1C or L2A.
             metadata_cache_dir: directory to cache product metadata files.
-            max_time_delta: maximum time before a query start time or after a
-                query end time to look for products. This is required due to the large
-                number of available products, and defaults to 30 days.
             sort_by: can be "cloud_cover", default arbitrary order; only has effect for
                 SpaceMode.WITHIN.
             harmonize: harmonize pixel values across different processing baselines,
                 see https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_S2_SR_HARMONIZED
+            context: the data source context.
         """  # noqa: E501
-        self.config = config
+        # If context is provided, we join the directory with the dataset path,
+        # otherwise we treat it directly as UPath.
+        if context.ds_path is not None:
+            self.metadata_cache_dir = join_upath(context.ds_path, metadata_cache_dir)
+        else:
+            self.metadata_cache_dir = UPath(metadata_cache_dir)
+
         self.modality = modality
-        self.metadata_cache_dir = metadata_cache_dir
-        self.max_time_delta = max_time_delta
         self.sort_by = sort_by
         self.harmonize = harmonize
 
         bucket_name = self.bucket_names[modality]
         self.bucket = boto3.resource("s3").Bucket(bucket_name)
 
-    @staticmethod
-    def from_config(config: LayerConfig, ds_path: UPath) -> "Sentinel2":
-        """Creates a new Sentinel2 instance from a configuration dictionary."""
-        assert isinstance(config, RasterLayerConfig)
-        d = config.data_source.config_dict
-        kwargs = dict(
-            config=config,
-            modality=Sentinel2Modality(d["modality"]),
-            metadata_cache_dir=join_upath(ds_path, d["metadata_cache_dir"]),
-        )
-
-        if "max_time_delta" in d:
-            kwargs["max_time_delta"] = timedelta(
-                seconds=pytimeparse.parse(d["max_time_delta"])
-            )
-        simple_optionals = ["sort_by", "harmonize"]
-        for k in simple_optionals:
-            if k in d:
-                kwargs[k] = d[k]
-
-        return Sentinel2(**kwargs)
-
     def _read_products(
-        self, needed_cell_months: set[tuple[str, int, int, int]]
+        self, needed_cell_months: set[tuple[str, int, int]]
     ) -> Generator[Sentinel2Item, None, None]:
         """Read productInfo.json files and yield relevant Sentinel2Items.
 
@@ -603,7 +556,7 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
 
     def get_items(
         self, geometries: list[STGeometry], query_config: QueryConfig
-    ) -> list[list[list[Item]]]:
+    ) -> list[list[list[Sentinel2Item]]]:
         """Get a list of items in the data source intersecting the given geometries.
 
         Args:
@@ -626,14 +579,14 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
                 raise ValueError(
                     "Sentinel2 on AWS requires geometry time ranges to be set"
                 )
-            for cell_id in rslearn.utils.mgrs.for_each_cell(wgs84_geometry.shp.bounds):
+            for cell_id in get_sentinel2_tiles(wgs84_geometry, self.metadata_cache_dir):
                 for ts in daterange(
-                    wgs84_geometry.time_range[0] - self.max_time_delta,
-                    wgs84_geometry.time_range[1] + self.max_time_delta,
+                    wgs84_geometry.time_range[0],
+                    wgs84_geometry.time_range[1],
                 ):
                     needed_cell_months.add((cell_id, ts.year, ts.month))
 
-        items_by_cell = {}
+        items_by_cell: dict[str, list[Sentinel2Item]] = {}
         for item in self._read_products(needed_cell_months):
             cell_id = "".join(item.blob_path.split("/")[1:4])
             if cell_id not in items_by_cell:
@@ -643,7 +596,7 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
         groups = []
         for geometry, wgs84_geometry in zip(geometries, wgs84_geometries):
             items = []
-            for cell_id in rslearn.utils.mgrs.for_each_cell(wgs84_geometry.shp.bounds):
+            for cell_id in get_sentinel2_tiles(wgs84_geometry, self.metadata_cache_dir):
                 for item in items_by_cell.get(cell_id, []):
                     try:
                         item_geom = item.geometry.to_projection(geometry.projection)
@@ -666,7 +619,7 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
 
         return groups
 
-    def get_item_by_name(self, name: str) -> Item:
+    def get_item_by_name(self, name: str) -> Sentinel2Item:
         """Gets an item by name."""
         # Product name is like:
         #   S2B_MSIL1C_20240201T230819_N0510_R015_T51CWM_20240202T012755.
@@ -685,12 +638,14 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
                 return item
         raise ValueError(f"item {name} not found")
 
-    def deserialize_item(self, serialized_item: Any) -> Item:
+    def deserialize_item(self, serialized_item: Any) -> Sentinel2Item:
         """Deserializes an item from JSON-decoded data."""
         assert isinstance(serialized_item, dict)
         return Sentinel2Item.deserialize(serialized_item)
 
-    def retrieve_item(self, item: Item) -> Generator[tuple[str, BinaryIO], None, None]:
+    def retrieve_item(
+        self, item: Sentinel2Item
+    ) -> Generator[tuple[str, BinaryIO], None, None]:
         """Retrieves the rasters corresponding to an item as file streams."""
         for fname, _ in self.band_fnames[self.modality]:
             buf = io.BytesIO()
@@ -701,7 +656,7 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
             yield (fname, buf)
 
     def _get_harmonize_callback(
-        self, item: Item
+        self, item: Sentinel2Item
     ) -> Callable[[npt.NDArray], npt.NDArray] | None:
         """Gets the harmonization callback for the given item.
 
@@ -715,6 +670,8 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
             return None
         # Search metadata XML for the RADIO_ADD_OFFSET tag.
         # This contains the per-band offset, but we assume all bands have the same offset.
+        if item.geometry.time_range is None:
+            raise ValueError("Sentinel2 on AWS requires geometry time ranges to be set")
         ts = item.geometry.time_range[0]
         metadata_fname = (
             f"products/{ts.year}/{ts.month}/{ts.day}/{item.name}/metadata.xml"
@@ -724,13 +681,15 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
             metadata_fname, buf, ExtraArgs={"RequestPayer": "requester"}
         )
         buf.seek(0)
-        tree = ET.ElementTree(ET.fromstring(buf.getvalue()))
+        tree: ET.ElementTree[ET.Element[str]] = ET.ElementTree(
+            ET.fromstring(buf.getvalue())
+        )
         return get_harmonize_callback(tree)
 
     def ingest(
         self,
-        tile_store: TileStore,
-        items: list[Item],
+        tile_store: TileStoreWithLayer,
+        items: list[Sentinel2Item],
         geometries: list[list[STGeometry]],
     ) -> None:
         """Ingest items into the given tile store.
@@ -740,42 +699,43 @@ class Sentinel2(ItemLookupDataSource, RetrieveItemDataSource):
             items: the items to ingest
             geometries: a list of geometries needed for each item
         """
-        for item, cur_geometries in zip(items, geometries):
-            harmonize_callback = self._get_harmonize_callback(item)
-
+        for item in items:
             for fname, band_names in self.band_fnames[self.modality]:
-                cur_tile_store = PrefixedTileStore(
-                    tile_store, (item.name, "_".join(band_names))
-                )
-                needed_projections = get_needed_projections(
-                    cur_tile_store, band_names, self.config.band_sets, cur_geometries
-                )
-                if not needed_projections:
+                if tile_store.is_raster_ready(item.name, band_names):
                     continue
 
-                buf = io.BytesIO()
-                try:
-                    self.bucket.download_fileobj(
-                        item.blob_path + fname,
-                        buf,
-                        ExtraArgs={"RequestPayer": "requester"},
-                    )
-                except Exception as e:
-                    # TODO: sometimes for some reason object doesn't exist
-                    # we should probably investigate further why it happens
-                    # and then should create the layer here and mark it completed
-                    print(
-                        f"warning: got error {e} downloading {item.blob_path + fname}"
-                    )
-                    continue
-                buf.seek(0)
-                with rasterio.open(buf) as raster:
-                    for projection in needed_projections:
-                        ingest_raster(
-                            tile_store=cur_tile_store,
-                            raster=raster,
-                            projection=projection,
-                            time_range=item.geometry.time_range,
-                            layer_config=self.config,
-                            array_callback=harmonize_callback,
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    local_fname = os.path.join(tmp_dir, fname.split("/")[-1])
+
+                    try:
+                        self.bucket.download_file(
+                            item.blob_path + fname,
+                            local_fname,
+                            ExtraArgs={"RequestPayer": "requester"},
+                        )
+                    except Exception as e:
+                        # TODO: sometimes for some reason object doesn't exist
+                        # we should probably investigate further why it happens
+                        # and then should create the layer here and mark it completed
+                        print(
+                            f"warning: got error {e} downloading {item.blob_path + fname}"
+                        )
+                        continue
+
+                    harmonize_callback = self._get_harmonize_callback(item)
+
+                    if harmonize_callback is not None:
+                        # In this case we need to read the array, convert the pixel
+                        # values, and pass modified array directly to the TileStore.
+                        with rasterio.open(local_fname) as src:
+                            array = src.read()
+                            projection, bounds = get_raster_projection_and_bounds(src)
+                        array = harmonize_callback(array)
+                        tile_store.write_raster(
+                            item.name, band_names, projection, bounds, array
+                        )
+
+                    else:
+                        tile_store.write_raster_file(
+                            item.name, band_names, UPath(local_fname)
                         )

rslearn/data_sources/aws_sentinel1.py

@@ -0,0 +1,131 @@
+"""Data source for Sentinel-1 on AWS."""
+
+import os
+import tempfile
+from typing import Any
+
+import boto3
+from upath import UPath
+
+from rslearn.data_sources.copernicus import (
+    CopernicusItem,
+    Sentinel1OrbitDirection,
+    Sentinel1Polarisation,
+    Sentinel1ProductType,
+)
+from rslearn.data_sources.copernicus import Sentinel1 as CopernicusSentinel1
+from rslearn.log_utils import get_logger
+from rslearn.tile_stores import TileStore, TileStoreWithLayer
+from rslearn.utils.geometry import STGeometry
+
+from .data_source import DataSource, DataSourceContext, QueryConfig
+
+WRS2_GRID_SIZE = 1.0
+
+logger = get_logger(__name__)
+
+
+class Sentinel1(DataSource, TileStore):
+    """A data source for Sentinel-1 GRD imagery on AWS.
+
+    Specifically, uses the sentinel-s1-l1c S3 bucket maintained by Sinergise. See
+    https://aws.amazon.com/marketplace/pp/prodview-uxrsbvhd35ifw for details about the
+    bucket.
+
+    We use the Copernicus API for metadata search. So the bucket is only used for
+    downloading the images.
+
+    Currently, it only supports GRD IW DV scenes.
+    """
+
+    bucket_name = "sentinel-s1-l1c"
+    bands = ["vv", "vh"]
+
+    def __init__(
+        self,
+        orbit_direction: Sentinel1OrbitDirection | None = None,
+        context: DataSourceContext = DataSourceContext(),
+    ) -> None:
+        """Initialize a new Sentinel1 instance.
+
+        Args:
+            orbit_direction: optional orbit direction to filter by.
+            context: the data source context.
+        """
+        self.client = boto3.client("s3")
+        self.bucket = boto3.resource("s3").Bucket(self.bucket_name)
+        self.sentinel1 = CopernicusSentinel1(
+            product_type=Sentinel1ProductType.IW_GRDH,
+            polarisation=Sentinel1Polarisation.VV_VH,
+            orbit_direction=orbit_direction,
+        )
+
+    def get_items(
+        self, geometries: list[STGeometry], query_config: QueryConfig
+    ) -> list[list[list[CopernicusItem]]]:
+        """Get a list of items in the data source intersecting the given geometries.
+
+        Args:
+            geometries: the spatiotemporal geometries
+            query_config: the query configuration
+
+        Returns:
+            List of groups of items that should be retrieved for each geometry.
+        """
+        return self.sentinel1.get_items(geometries, query_config)
+
+    def get_item_by_name(self, name: str) -> CopernicusItem:
+        """Gets an item by name."""
+        return self.sentinel1.get_item_by_name(name)
+
+    def deserialize_item(self, serialized_item: Any) -> CopernicusItem:
+        """Deserializes an item from JSON-decoded data."""
+        assert isinstance(serialized_item, dict)
+        return CopernicusItem.deserialize(serialized_item)
+
+    def ingest(
+        self,
+        tile_store: TileStoreWithLayer,
+        items: list[CopernicusItem],
+        geometries: list[list[STGeometry]],
+    ) -> None:
+        """Ingest items into the given tile store.
+
+        Args:
+            tile_store: the tile store to ingest into
+            items: the items to ingest
+            geometries: a list of geometries needed for each item
+        """
+        for item in items:
+            for band in self.bands:
+                band_names = [band]
+                if tile_store.is_raster_ready(item.name, band_names):
+                    continue
+
+                # Item name is like "S1C_IW_GRDH_1SDV_20250528T172106_20250528T172131_002534_00545C_B433.SAFE".
+                item_name_prefix = item.name.split(".")[0]
+                time_str = item_name_prefix.split("_")[4]
+                if len(time_str) != 15:
+                    raise ValueError(
+                        f"expected 15-character time string but got {time_str}"
+                    )
+                # We convert to int here since path in bucket isn't padded with leading 0s.
+                year = int(time_str[0:4])
+                month = int(time_str[4:6])
+                day = int(time_str[6:8])
+                blob_path = f"GRD/{year}/{month}/{day}/IW/DV/{item_name_prefix}/measurement/iw-{band}.tiff"
+
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    fname = os.path.join(tmp_dir, f"{band}.tif")
+                    try:
+                        self.bucket.download_file(
+                            blob_path,
+                            fname,
+                            ExtraArgs={"RequestPayer": "requester"},
+                        )
+                    except:
+                        logger.error(
+                            f"encountered error while downloading s3://{self.bucket_name}/{blob_path}"
+                        )
+                        raise
+                    tile_store.write_raster_file(item.name, band_names, UPath(fname))