rslearn 0.0.8__tar.gz → 0.0.9__tar.gz

{rslearn-0.0.8/rslearn.egg-info → rslearn-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rslearn
-Version: 0.0.8
+Version: 0.0.9
 Summary: A library for developing remote sensing datasets and models
 Author: OlmoEarth Team
 License:                                  Apache License
@@ -212,7 +212,6 @@ Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: boto3>=1.39
-Requires-Dist: class_registry>=2.1
 Requires-Dist: fiona>=1.10
 Requires-Dist: fsspec>=2025.9.0
 Requires-Dist: jsonargparse>=4.35.0
@@ -284,6 +283,7 @@ Quick links:
 - [Examples](docs/Examples.md) contains more examples, including customizing different
   stages of rslearn with additional code.
 - [DatasetConfig](docs/DatasetConfig.md) documents the dataset configuration file.
+- [ModelConfig](docs/ModelConfig.md) documents the model configuration file.
 
 
 Setup

{rslearn-0.0.8 → rslearn-0.0.9}/README.md

@@ -21,6 +21,7 @@ Quick links:
 - [Examples](docs/Examples.md) contains more examples, including customizing different
   stages of rslearn with additional code.
 - [DatasetConfig](docs/DatasetConfig.md) documents the dataset configuration file.
+- [ModelConfig](docs/ModelConfig.md) documents the model configuration file.
 
 
 Setup

{rslearn-0.0.8 → rslearn-0.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "rslearn"
-version = "0.0.8"
+version = "0.0.9"
 description = "A library for developing remote sensing datasets and models"
 authors = [
     { name = "OlmoEarth Team" },
@@ -10,7 +10,6 @@ license = {file = "LICENSE"}
 requires-python = ">=3.11"
 dependencies = [
     "boto3>=1.39",
-    "class_registry>=2.1",
     "fiona>=1.10",
     "fsspec>=2025.9.0", # this is used both directly and indirectly (via universal_pathlib) in our code
     "jsonargparse>=4.35.0",

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/data_sources/local_files.py

@@ -2,12 +2,12 @@
 
 import functools
 import json
+from collections.abc import Callable
 from typing import Any, Generic, TypeVar
 
 import fiona
 import shapely
 import shapely.geometry
-from class_registry import ClassRegistry
 from rasterio.crs import CRS
 from upath import UPath
 
@@ -23,7 +23,24 @@ from rslearn.utils.geometry import Projection, STGeometry, get_global_geometry
 from .data_source import DataSource, Item, QueryConfig
 
 logger = get_logger("__name__")
-Importers = ClassRegistry()
+_ImporterT = TypeVar("_ImporterT", bound="Importer")
+
+
+class _ImporterRegistry(dict[str, type["Importer"]]):
+    """Registry for Importer classes."""
+
+    def register(self, name: str) -> Callable[[type[_ImporterT]], type[_ImporterT]]:
+        """Decorator to register an importer class."""
+
+        def decorator(cls: type[_ImporterT]) -> type[_ImporterT]:
+            self[name] = cls
+            return cls
+
+        return decorator
+
+
+Importers = _ImporterRegistry()
+
 
 ItemType = TypeVar("ItemType", bound=Item)
 LayerConfigType = TypeVar("LayerConfigType", bound=LayerConfig)
@@ -425,7 +442,7 @@ class LocalFiles(DataSource):
         """
         self.config = config
 
-        self.importer = Importers[config.layer_type.value]
+        self.importer = Importers[config.layer_type.value]()
         self.src_dir = src_dir
 
     @staticmethod

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/data_sources/planetary_computer.py

@@ -83,6 +83,10 @@ class PlanetaryComputer(DataSource, TileStore):
 
     STAC_ENDPOINT = "https://planetarycomputer.microsoft.com/api/stac/v1"
 
+    # Default threshold for recreating the STAC client to prevent memory leaks
+    # from the pystac Catalog's resolved objects cache growing unbounded
+    DEFAULT_MAX_ITEMS_PER_CLIENT = 1000
+
     def __init__(
         self,
         collection_name: str,
@@ -93,6 +97,7 @@ class PlanetaryComputer(DataSource, TileStore):
         timeout: timedelta = timedelta(seconds=10),
         skip_items_missing_assets: bool = False,
         cache_dir: UPath | None = None,
+        max_items_per_client: int | None = None,
     ):
         """Initialize a new PlanetaryComputer instance.
 
@@ -109,6 +114,9 @@ class PlanetaryComputer(DataSource, TileStore):
             cache_dir: optional directory to cache items by name, including asset URLs.
                 If not set, there will be no cache and instead STAC requests will be
                 needed each time.
+            max_items_per_client: number of STAC items to process before recreating
+                the client to prevent memory leaks from the resolved objects cache.
+                Defaults to DEFAULT_MAX_ITEMS_PER_CLIENT.
         """
         self.collection_name = collection_name
         self.asset_bands = asset_bands
@@ -118,12 +126,15 @@ class PlanetaryComputer(DataSource, TileStore):
         self.timeout = timeout
         self.skip_items_missing_assets = skip_items_missing_assets
         self.cache_dir = cache_dir
+        self.max_items_per_client = (
+            max_items_per_client or self.DEFAULT_MAX_ITEMS_PER_CLIENT
+        )
 
         if self.cache_dir is not None:
             self.cache_dir.mkdir(parents=True, exist_ok=True)
 
         self.client: pystac_client.Client | None = None
-        self.collection: pystac_client.CollectionClient | None = None
+        self._client_item_count = 0
 
     @staticmethod
     def from_config(config: RasterLayerConfig, ds_path: UPath) -> "PlanetaryComputer":
@@ -142,7 +153,12 @@ class PlanetaryComputer(DataSource, TileStore):
         if "cache_dir" in d:
             kwargs["cache_dir"] = join_upath(ds_path, d["cache_dir"])
 
-        simple_optionals = ["query", "sort_by", "sort_ascending"]
+        simple_optionals = [
+            "query",
+            "sort_by",
+            "sort_ascending",
+            "max_items_per_client",
+        ]
         for k in simple_optionals:
             if k in d:
                 kwargs[k] = d[k]
@@ -151,20 +167,40 @@ class PlanetaryComputer(DataSource, TileStore):
 
     def _load_client(
         self,
-    ) -> tuple[pystac_client.Client, pystac_client.CollectionClient]:
+    ) -> pystac_client.Client:
         """Lazily load pystac client.
 
         We don't load it when creating the data source because it takes time and caller
         may not be calling get_items. Additionally, loading it during the get_items
         call enables leveraging the retry loop functionality in
         prepare_dataset_windows.
-        """
-        if self.client is not None:
-            return self.client, self.collection
 
+        Note: We periodically recreate the client to prevent memory leaks from the
+        pystac Catalog's resolved objects cache, which grows unbounded as STAC items
+        are deserialized and cached. The cache cannot be cleared or disabled.
+        """
+        if self.client is None:
+            logger.info("Creating initial STAC client")
+            self.client = pystac_client.Client.open(self.STAC_ENDPOINT)
+            return self.client
+
+        if self._client_item_count < self.max_items_per_client:
+            return self.client
+
+        # Recreate client to clear the resolved objects cache
+        current_client = self.client
+        logger.debug(
+            "Recreating STAC client after processing %d items (threshold: %d)",
+            self._client_item_count,
+            self.max_items_per_client,
+        )
+        client_root = current_client.get_root()
+        client_root.clear_links()
+        client_root.clear_items()
+        client_root.clear_children()
+        self._client_item_count = 0
         self.client = pystac_client.Client.open(self.STAC_ENDPOINT)
-        self.collection = self.client.get_collection(self.collection_name)
-        return self.client, self.collection
+        return self.client
 
     def _stac_item_to_item(self, stac_item: pystac.Item) -> PlanetaryComputerItem:
         shp = shapely.geometry.shape(stac_item.geometry)
@@ -210,10 +246,26 @@ class PlanetaryComputer(DataSource, TileStore):
 
         # No cache or not in cache, so we need to make the STAC request.
         logger.debug("Getting STAC item {name}")
-        _, collection = self._load_client()
-        stac_item = collection.get_item(name)
+        client = self._load_client()
+
+        search_result = client.search(ids=[name], collections=[self.collection_name])
+        stac_items = list(search_result.items())
+
+        if not stac_items:
+            raise ValueError(
+                f"Item {name} not found in collection {self.collection_name}"
+            )
+        if len(stac_items) > 1:
+            raise ValueError(
+                f"Multiple items found for ID {name} in collection {self.collection_name}"
+            )
+
+        stac_item = stac_items[0]
         item = self._stac_item_to_item(stac_item)
 
+        # Track items processed for client recreation threshold (after deserialization)
+        self._client_item_count += 1
+
         # Finally we cache it if cache_dir is set.
         if cache_fname is not None:
             with cache_fname.open("w") as f:
@@ -233,7 +285,7 @@ class PlanetaryComputer(DataSource, TileStore):
         Returns:
             List of groups of items that should be retrieved for each geometry.
         """
-        client, _ = self._load_client()
+        client = self._load_client()
 
         groups = []
         for geometry in geometries:
@@ -247,7 +299,9 @@ class PlanetaryComputer(DataSource, TileStore):
                 datetime=wgs84_geometry.time_range,
                 query=self.query,
             )
-            stac_items = [item for item in result.item_collection()]
+            stac_items = [item for item in result.items()]
+            # Track items processed for client recreation threshold (after deserialization)
+            self._client_item_count += len(stac_items)
             logger.debug("STAC search yielded %d items", len(stac_items))
 
             if self.skip_items_missing_assets:
@@ -580,7 +634,13 @@ class Sentinel2(PlanetaryComputer):
         if "cache_dir" in d:
             kwargs["cache_dir"] = join_upath(ds_path, d["cache_dir"])
 
-        simple_optionals = ["harmonize", "query", "sort_by", "sort_ascending"]
+        simple_optionals = [
+            "harmonize",
+            "query",
+            "sort_by",
+            "sort_ascending",
+            "max_items_per_client",
+        ]
         for k in simple_optionals:
             if k in d:
                 kwargs[k] = d[k]
@@ -756,7 +816,12 @@ class Sentinel1(PlanetaryComputer):
         if "cache_dir" in d:
             kwargs["cache_dir"] = join_upath(ds_path, d["cache_dir"])
 
-        simple_optionals = ["query", "sort_by", "sort_ascending"]
+        simple_optionals = [
+            "query",
+            "sort_by",
+            "sort_ascending",
+            "max_items_per_client",
+        ]
         for k in simple_optionals:
             if k in d:
                 kwargs[k] = d[k]

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/dataset/manage.py

@@ -396,9 +396,9 @@ def materialize_window(
         )
 
         if dataset.materializer_name:
-            materializer = Materializers[dataset.materializer_name]
+            materializer = Materializers[dataset.materializer_name]()
         else:
-            materializer = Materializers[layer_cfg.layer_type.value]
+            materializer = Materializers[layer_cfg.layer_type.value]()
 
         retry(
             fn=lambda: materializer.materialize(

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/dataset/materialize.py

@@ -1,10 +1,10 @@
 """Classes to implement dataset materialization."""
 
+from collections.abc import Callable
 from typing import Any, Generic, TypeVar
 
 import numpy as np
 import numpy.typing as npt
-from class_registry import ClassRegistry
 from rasterio.enums import Resampling
 
 from rslearn.config import (
@@ -25,7 +25,26 @@ from rslearn.utils.vector_format import load_vector_format
 from .remap import Remapper, load_remapper
 from .window import Window
 
-Materializers = ClassRegistry()
+_MaterializerT = TypeVar("_MaterializerT", bound="Materializer")
+
+
+class _MaterializerRegistry(dict[str, type["Materializer"]]):
+    """Registry for Materializer classes."""
+
+    def register(
+        self, name: str
+    ) -> Callable[[type[_MaterializerT]], type[_MaterializerT]]:
+        """Decorator to register a materializer class."""
+
+        def decorator(cls: type[_MaterializerT]) -> type[_MaterializerT]:
+            self[name] = cls
+            return cls
+
+        return decorator
+
+
+Materializers = _MaterializerRegistry()
+
 
 LayerConfigType = TypeVar("LayerConfigType", bound=LayerConfig)
 

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/dataset/remap.py

@@ -1,18 +1,42 @@
 """Classes to remap raster values."""
 
-from typing import Any
+from collections.abc import Callable
+from typing import Any, TypeVar
 
 import numpy as np
 import numpy.typing as npt
-from class_registry import ClassRegistry
 
-Remappers = ClassRegistry()
+_RemapperT = TypeVar("_RemapperT", bound="Remapper")
+
+
+class _RemapperRegistry(dict[str, type["Remapper"]]):
+    """Registry for Remapper classes."""
+
+    def register(self, name: str) -> Callable[[type[_RemapperT]], type[_RemapperT]]:
+        """Decorator to register a remapper class."""
+
+        def decorator(cls: type[_RemapperT]) -> type[_RemapperT]:
+            self[name] = cls
+            return cls
+
+        return decorator
+
+
+Remappers = _RemapperRegistry()
 """Registry of Remapper implementations."""
 
 
 class Remapper:
     """An abstract class that remaps pixel values based on layer configuration."""
 
+    def __init__(self, config: dict[str, Any]) -> None:
+        """Initialize a Remapper.
+
+        Args:
+            config: the config dict for this remapper.
+        """
+        pass
+
     def __call__(
         self, array: npt.NDArray[Any], dtype: npt.DTypeLike
     ) -> npt.NDArray[Any]:
@@ -67,4 +91,5 @@ class LinearRemapper(Remapper):
 
 def load_remapper(config: dict[str, Any]) -> Remapper:
     """Load a remapper from a configuration dictionary."""
-    return Remappers.get(config["name"], config=config)
+    cls = Remappers[config["name"]]
+    return cls(config)

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/models/dinov3.py

@@ -7,8 +7,8 @@ from typing import Any
 import torch
 import torchvision
 from einops import rearrange
-from torchvision.transforms import v2
 
+from rslearn.train.transforms.normalize import Normalize
 from rslearn.train.transforms.transform import Transform
 
 
@@ -139,15 +139,17 @@ class DinoV3Normalize(Transform):
         super().__init__()
         self.satellite = satellite
         if satellite:
-            self.normalize = v2.Normalize(
-                mean=(0.430, 0.411, 0.296),
-                std=(0.213, 0.156, 0.143),
-            )
+            mean = [0.430, 0.411, 0.296]
+            std = [0.213, 0.156, 0.143]
         else:
-            self.normalize = v2.Normalize(
-                mean=(0.485, 0.456, 0.406),
-                std=(0.229, 0.224, 0.225),
-            )
+            mean = [0.485, 0.456, 0.406]
+            std = [0.229, 0.224, 0.225]
+
+        self.normalize = Normalize(
+            [value * 255 for value in mean],
+            [value * 255 for value in std],
+            num_bands=3,
+        )
 
     def forward(
         self, input_dict: dict[str, Any], target_dict: dict[str, Any]
@@ -161,5 +163,4 @@ class DinoV3Normalize(Transform):
         Returns:
             normalized (input_dicts, target_dicts) tuple
         """
-        input_dict["image"] = self.normalize(input_dict["image"] / 255.0)
-        return input_dict, target_dict
+        return self.normalize(input_dict, target_dict)

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/models/galileo/galileo.py

@@ -2,6 +2,7 @@
 
 import math
 import tempfile
+from contextlib import nullcontext
 from enum import StrEnum
 from typing import Any, cast
 
@@ -63,6 +64,11 @@ pretrained_weights: dict[GalileoSize, str] = {
 
 DEFAULT_NORMALIZER = Normalizer()
 
+AUTOCAST_DTYPE_MAP = {
+    "bfloat16": torch.bfloat16,
+    "float32": torch.float32,
+}
+
 
 class GalileoModel(nn.Module):
     """Galileo backbones."""
@@ -85,6 +91,7 @@ class GalileoModel(nn.Module):
         size: GalileoSize,
         patch_size: int = 4,
         pretrained_path: str | UPath | None = None,
+        autocast_dtype: str | None = "bfloat16",
     ) -> None:
         """Initialize the Galileo model.
 
@@ -93,6 +100,7 @@ class GalileoModel(nn.Module):
             patch_size: The patch size to use.
             pretrained_path: the local path to the pretrained weights. Otherwise it is
                 downloaded and cached in temp directory.
+            autocast_dtype: which dtype to use for autocasting, or set None to disable.
         """
         super().__init__()
         if pretrained_path is None:
@@ -128,8 +136,14 @@ class GalileoModel(nn.Module):
             idx for idx, key in enumerate(SPACE_TIME_BANDS_GROUPS_IDX) if "S1" in key
         ]
 
+        self.size = size
         self.patch_size = patch_size
 
+        if autocast_dtype is not None:
+            self.autocast_dtype = AUTOCAST_DTYPE_MAP[autocast_dtype]
+        else:
+            self.autocast_dtype = None
+
     @staticmethod
     def to_cartesian(
         lat: float | np.ndarray | torch.Tensor, lon: float | np.ndarray | torch.Tensor
@@ -484,18 +498,31 @@ class GalileoModel(nn.Module):
             patch_size = h
         else:
             patch_size = self.patch_size
-        outputs = self.model(
-            s_t_x=galileo_input.s_t_x,
-            s_t_m=galileo_input.s_t_m,
-            sp_x=galileo_input.sp_x,
-            sp_m=galileo_input.sp_m,
-            t_x=galileo_input.t_x,
-            t_m=galileo_input.t_m,
-            st_x=galileo_input.st_x,
-            st_m=galileo_input.st_m,
-            months=galileo_input.months,
-            patch_size=patch_size,
-        )
+
+        # Decide context based on self.autocast_dtype.
+        device = galileo_input.s_t_x.device
+        if self.autocast_dtype is None:
+            context = nullcontext()
+        else:
+            assert device is not None
+            context = torch.amp.autocast(
+                device_type=device.type, dtype=self.autocast_dtype
+            )
+
+        with context:
+            outputs = self.model(
+                s_t_x=galileo_input.s_t_x,
+                s_t_m=galileo_input.s_t_m,
+                sp_x=galileo_input.sp_x,
+                sp_m=galileo_input.sp_m,
+                t_x=galileo_input.t_x,
+                t_m=galileo_input.t_m,
+                st_x=galileo_input.st_x,
+                st_m=galileo_input.st_m,
+                months=galileo_input.months,
+                patch_size=patch_size,
+            )
+
         if h == patch_size:
             # only one spatial patch, so we can just take an average
             # of all the tokens to output b c_g 1 1
@@ -515,3 +542,22 @@ class GalileoModel(nn.Module):
                     "b h w c_g d -> b c_g d h w",
                 ).mean(dim=1)
             ]
+
+    def get_backbone_channels(self) -> list:
+        """Returns the output channels of this model when used as a backbone.
+
+        The output channels is a list of (patch_size, depth) that corresponds
+        to the feature maps that the backbone returns.
+
+        Returns:
+            the output channels of the backbone as a list of (patch_size, depth) tuples.
+        """
+        if self.size == GalileoSize.BASE:
+            depth = 768
+        elif self.model_size == GalileoSize.TINY:
+            depth = 192
+        elif self.model_size == GalileoSize.NANO:
+            depth = 128
+        else:
+            raise ValueError(f"Invalid model size: {self.size}")
+        return [(self.patch_size, depth)]

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/models/galileo/single_file_galileo.py

@@ -1469,7 +1469,13 @@ class Encoder(GalileoBase):
             # we take the inverse of the mask because a value
             # of True indicates the value *should* take part in
             # attention
-            x = blk(x=x, y=None, attn_mask=~new_m.bool())
+            temp_mask = ~new_m.bool()
+            if temp_mask.all():
+                # if all the tokens are used in attention we can pass a None mask
+                # to the attention block
+                temp_mask = None
+
+            x = blk(x=x, y=None, attn_mask=temp_mask)
 
         if exit_ids_seq is not None:
             assert exited_tokens is not None

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/models/presto/presto.py

@@ -248,3 +248,14 @@ class Presto(nn.Module):
             output_features[batch_idx : batch_idx + self.pixel_batch_size] = output_b
 
         return [rearrange(output_features, "(b h w) d -> b d h w", h=h, w=w, b=b)]
+
+    def get_backbone_channels(self) -> list:
+        """Returns the output channels of this model when used as a backbone.
+
+        The output channels is a list of (patch_size, depth) that corresponds
+        to the feature maps that the backbone returns.
+
+        Returns:
+            the output channels of the backbone as a list of (patch_size, depth) tuples.
+        """
+        return [(1, 128)]

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/models/prithvi.py

@@ -173,6 +173,17 @@ class PrithviV2(nn.Module):
             features, num_timesteps
         )
 
+    def get_backbone_channels(self) -> list:
+        """Returns the output channels of this model when used as a backbone.
+
+        The output channels is a list of (patch_size, depth) that corresponds
+        to the feature maps that the backbone returns.
+
+        Returns:
+            the output channels of the backbone as a list of (patch_size, depth) tuples.
+        """
+        return [(1, 1024)]
+
 
 class PrithviNormalize(Transform):
     """Normalize inputs using Prithvi normalization.

rslearn-0.0.9/rslearn/models/registry.py

@@ -0,0 +1,22 @@
+"""Model registry."""
+
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+_ModelT = TypeVar("_ModelT")
+
+
+class _ModelRegistry(dict[str, type[Any]]):
+    """Registry for Model classes."""
+
+    def register(self, name: str) -> Callable[[type[_ModelT]], type[_ModelT]]:
+        """Decorator to register a model class."""
+
+        def decorator(cls: type[_ModelT]) -> type[_ModelT]:
+            self[name] = cls
+            return cls
+
+        return decorator
+
+
+Models = _ModelRegistry()

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/tile_stores/default.py

@@ -130,10 +130,12 @@ class DefaultTileStore(TileStore):
         """
         raster_dir = self._get_raster_dir(layer_name, item_name, bands)
         for fname in raster_dir.iterdir():
-            # Ignore completed sentinel files as well as temporary files created by
+            # Ignore completed sentinel files, bands files, as well as temporary files created by
             # open_atomic (in case this tile store is on local filesystem).
             if fname.name == COMPLETED_FNAME:
                 continue
+            if fname.name == BANDS_FNAME:
+                continue
             if ".tmp." in fname.name:
                 continue
             return fname

{rslearn-0.0.8 → rslearn-0.0.9}/rslearn/train/transforms/transform.py

@@ -54,7 +54,7 @@ def read_selector(
         the item specified by the selector
     """
     d, selector = get_dict_and_subselector(input_dict, target_dict, selector)
-    parts = selector.split("/")
+    parts = selector.split("/") if selector else []
     cur = d
     for part in parts:
         cur = cur[part]
@@ -76,11 +76,28 @@ def write_selector(
         v: the value to write
     """
     d, selector = get_dict_and_subselector(input_dict, target_dict, selector)
-    parts = selector.split("/")
-    cur = d
-    for part in parts[:-1]:
-        cur = cur[part]
-    cur[parts[-1]] = v
+    if selector:
+        parts = selector.split("/")
+        cur = d
+        for part in parts[:-1]:
+            cur = cur[part]
+        cur[parts[-1]] = v
+    else:
+        # If the selector references the input or target dictionary directly, then we
+        # have a special case where instead of overwriting with v, we replace the keys
+        # with those in v. v must be a dictionary here, not a tensor, since otherwise
+        # it wouldn't match the type of the input or target dictionary.
+        if not isinstance(v, dict):
+            raise ValueError(
+                "when directly specifying the input or target dict, expected the value to be a dict"
+            )
+        if d == v:
+            # This may happen if the writer did not make a copy of the dictionary. In
+            # this case the code below would not update d correctly since it would also
+            # clear v.
+            return
+        d.clear()
+        d.update(v)
 
 
 class Transform(torch.nn.Module):