tilebox-storage 0.46.0__py3-none-any.whl → 0.48.0__py3-none-any.whl

tilebox/storage/__init__.py

@@ -2,6 +2,7 @@ from pathlib import Path
 
 from tilebox.storage.aio import ASFStorageClient as _ASFStorageClient
 from tilebox.storage.aio import CopernicusStorageClient as _CopernicusStorageClient
+from tilebox.storage.aio import LocalFileSystemStorageClient as _LocalFileSystemStorageClient
 from tilebox.storage.aio import UmbraStorageClient as _UmbraStorageClient
 from tilebox.storage.aio import USGSLandsatStorageClient as _USGSLandsatStorageClient
 
@@ -66,3 +67,14 @@ class USGSLandsatStorageClient(_USGSLandsatStorageClient):
         """
         super().__init__(cache_directory)
         self._syncify()
+
+
+class LocalFileSystemStorageClient(_LocalFileSystemStorageClient):
+    def __init__(self, root: Path) -> None:
+        """A tilebox storage client for accessing data on a local file system, or a mounted network file system.
+
+        Args:
+            root: The root directory of the file system to access.
+        """
+        super().__init__(root)
+        self._syncify()

tilebox/storage/aio.py

@@ -7,6 +7,7 @@ import zipfile
 from asyncio import Queue, QueueEmpty
 from collections.abc import AsyncIterator
 from pathlib import Path
+from pathlib import PurePosixPath as ObjectPath
 from typing import Any, TypeAlias
 
 import anyio
@@ -23,13 +24,14 @@ from _tilebox.grpc.aio.syncify import Syncifiable
 from tilebox.storage.granule import (
     ASFStorageGranule,
     CopernicusStorageGranule,
+    LocationStorageGranule,
     UmbraStorageGranule,
     USGSLandsatStorageGranule,
 )
 from tilebox.storage.providers import login
 
 try:
-    from IPython.display import HTML, Image, display  # type: ignore[assignment]
+    from IPython.display import HTML, Image, display
 except ImportError:
     # IPython is not available, so we can't display the quicklook image
     # but let's define stubs for the type checker
@@ -240,6 +242,10 @@ def _display_quicklook(image_data: bytes | Path, width: int, height: int, image_
 
 
 class StorageClient(Syncifiable):
+    """Base class for all storage clients."""
+
+
+class CachingStorageClient(StorageClient):
     def __init__(self, cache_directory: Path | None) -> None:
         self._cache = cache_directory
 
@@ -259,8 +265,8 @@ class StorageClient(Syncifiable):
 
 async def list_object_paths(store: ObjectStore, prefix: str) -> list[str]:
     objects = await obs.list(store, prefix).collect_async()
-    prefix_path = Path(prefix)
-    return sorted(str(Path(obj["path"]).relative_to(prefix_path)) for obj in objects)
+    prefix_path = ObjectPath(prefix)
+    return sorted(str(ObjectPath(obj["path"]).relative_to(prefix_path)) for obj in objects)
 
 
 async def download_objects(  # noqa: PLR0913
@@ -299,7 +305,7 @@ async def _download_worker(
 async def _download_object(
     store: ObjectStore, prefix: str, obj: str, output_dir: Path, show_progress: bool = True
 ) -> Path:
-    key = str(Path(prefix) / obj)
+    key = str(ObjectPath(prefix) / obj)
     output_path = output_dir / obj
     if output_path.exists():  # already cached
         return output_path
@@ -322,7 +328,7 @@ async def _download_object(
     return output_path
 
 
-class ASFStorageClient(StorageClient):
+class ASFStorageClient(CachingStorageClient):
     def __init__(self, user: str, password: str, cache_directory: Path = Path.home() / ".cache" / "tilebox") -> None:
         """A tilebox storage client that downloads data from the Alaska Satellite Facility.
 
@@ -414,7 +420,7 @@ class ASFStorageClient(StorageClient):
         """
         granule = ASFStorageGranule.from_data(datapoint)
         if Image is None:
-            raise ImportError("IPython is not available, please use download_preview instead.")
+            raise ImportError("IPython is not available, please use download_quicklook instead.")
         quicklook = await self._download_quicklook(datapoint)
         _display_quicklook(quicklook, width, height, f"<code>Image {quicklook.name} © ASF {granule.time.year}</code>")
 
@@ -438,7 +444,7 @@ def _umbra_s3_prefix(datapoint: xr.Dataset | UmbraStorageGranule) -> str:
     return f"sar-data/tasks/{granule.location}/"
 
 
-class UmbraStorageClient(StorageClient):
+class UmbraStorageClient(CachingStorageClient):
     _STORAGE_PROVIDER = "Umbra"
     _BUCKET = "umbra-open-data-catalog"
     _REGION = "us-west-2"
@@ -538,7 +544,7 @@ def _copernicus_s3_prefix(datapoint: xr.Dataset | CopernicusStorageGranule) -> s
     return granule.location.removeprefix("/eodata/")
 
 
-class CopernicusStorageClient(StorageClient):
+class CopernicusStorageClient(CachingStorageClient):
     _STORAGE_PROVIDER = "CopernicusDataspace"
     _BUCKET = "eodata"
     _ENDPOINT_URL = "https://eodata.dataspace.copernicus.eu"
@@ -609,7 +615,7 @@ class CopernicusStorageClient(StorageClient):
         granule = CopernicusStorageGranule.from_data(datapoint)
         # special handling for Sentinel-5P, where the location is not a folder but a single file
         if granule.location.endswith(".nc"):
-            return [Path(granule.granule_name).name]
+            return [str(ObjectPath(granule.granule_name))]
 
         return await list_object_paths(self._store, _copernicus_s3_prefix(granule))
 
@@ -723,7 +729,7 @@ class CopernicusStorageClient(StorageClient):
             ValueError: If no quicklook is available for the given datapoint.
         """
         if Image is None:
-            raise ImportError("IPython is not available, please use download_preview instead.")
+            raise ImportError("IPython is not available, please use download_quicklook instead.")
         granule = CopernicusStorageGranule.from_data(datapoint)
         quicklook = await self._download_quicklook(granule)
         _display_quicklook(quicklook, width, height, f"<code>{granule.granule_name} © ESA {granule.time.year}</code>")
@@ -749,7 +755,7 @@ def _landsat_s3_prefix(datapoint: xr.Dataset | USGSLandsatStorageGranule) -> str
     return granule.location.removeprefix("s3://usgs-landsat/")
 
 
-class USGSLandsatStorageClient(StorageClient):
+class USGSLandsatStorageClient(CachingStorageClient):
     """
     A client for downloading USGS Landsat data from the usgs-landsat and usgs-landsat-ard S3 bucket.
 
@@ -882,7 +888,7 @@ class USGSLandsatStorageClient(StorageClient):
             ValueError: If no quicklook is available for the given datapoint.
         """
         if Image is None:
-            raise ImportError("IPython is not available, please use download_preview instead.")
+            raise ImportError("IPython is not available, please use download_quicklook instead.")
         quicklook = await self._download_quicklook(datapoint)
         _display_quicklook(quicklook, width, height, f"<code>Image {quicklook.name} © USGS</code>")
 
@@ -900,3 +906,77 @@ class USGSLandsatStorageClient(StorageClient):
 
         await download_objects(self._store, prefix, [granule.thumbnail], output_folder, show_progress=False)
         return output_folder / granule.thumbnail
+
+
+class LocalFileSystemStorageClient(StorageClient):
+    def __init__(self, root: Path) -> None:
+        """A tilebox storage client for accessing data on a local file system, or a mounted network file system.
+
+        Args:
+            root: The root directory of the file system to access.
+        """
+        super().__init__()
+        self._root = Path(root)
+
+    async def list_objects(self, datapoint: xr.Dataset | LocationStorageGranule) -> list[str]:
+        """List all available objects for a given datapoint."""
+        granule = LocationStorageGranule.from_data(datapoint)
+        granule_path = self._root / granule.location
+        return [p.relative_to(granule_path).as_posix() for p in granule_path.rglob("**/*") if p.is_file()]
+
+    async def download(
+        self,
+        datapoint: xr.Dataset | LocationStorageGranule,
+    ) -> Path:
+        """No-op download method, as the data is already on the local file system.
+
+        Args:
+            datapoint: The datapoint to locate the data for in the local file system.
+
+        Returns:
+            The path to the data on the local file system.
+        """
+        granule = LocationStorageGranule.from_data(datapoint)
+        granule_path = self._root / granule.location
+        if not granule_path.exists():
+            raise ValueError(f"Data not found on the local file system: {granule_path}")
+        return granule_path
+
+    async def _download_quicklook(self, datapoint: xr.Dataset | LocationStorageGranule) -> Path:
+        granule = LocationStorageGranule.from_data(datapoint)
+        if granule.thumbnail is None:
+            raise ValueError(f"No quicklook available for {granule.location}")
+        quicklook_path = self._root / granule.thumbnail
+        if not quicklook_path.exists():
+            raise ValueError(f"Quicklook not found on the local file system: {quicklook_path}")
+        return quicklook_path
+
+    async def download_quicklook(self, datapoint: xr.Dataset | LocationStorageGranule) -> Path:
+        """No-op download_quicklook method, as the quicklook image is already on the local file system.
+
+        Args:
+            datapoint: The datapoint to locate the quicklook image for in the local file system.
+
+        Returns:
+            The path to the data on the local file system.
+
+        Raises:
+            ValueError: If no quicklook image is available for the given datapoint, or if the quicklook image is not
+                found on the local file system.
+        """
+        return await self._download_quicklook(datapoint)
+
+    async def quicklook(
+        self, datapoint: xr.Dataset | LocationStorageGranule, width: int = 600, height: int = 600
+    ) -> None:
+        """Display the quicklook image for a given datapoint.
+
+        Args:
+            datapoint: The datapoint to display the quicklook for.
+            width: Display width of the image in pixels. Defaults to 600.
+            height: Display height of the image in pixels. Defaults to 600.
+        """
+        quicklook_path = await self._download_quicklook(datapoint)
+        if Image is None:
+            raise ImportError("IPython is not available, please use download_quicklook instead.")
+        _display_quicklook(quicklook_path, width, height, None)

tilebox/storage/granule.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from datetime import datetime
-from pathlib import Path
+from pathlib import PurePosixPath as ObjectPath
 
 import xarray as xr
 
@@ -103,7 +103,7 @@ def _thumbnail_relative_to_eodata_location(thumbnail_url: str, location: str) ->
     url_path = thumbnail_url.split("?path=")[-1]
     url_path = url_path.removeprefix("/")
     location = location.removeprefix("/eodata/")
-    return str(Path(url_path).relative_to(location))
+    return str(ObjectPath(url_path).relative_to(location))
 
 
 @dataclass
@@ -183,3 +183,28 @@ class USGSLandsatStorageGranule:
             dataset.location.item().replace("s3://usgs-landsat-ard/", "s3://usgs-landsat/"),
             thumbnail,
         )
+
+
+@dataclass
+class LocationStorageGranule:
+    location: str
+    thumbnail: str | None = None
+
+    @classmethod
+    def from_data(cls, dataset: "xr.Dataset | LocationStorageGranule") -> "LocationStorageGranule":
+        """Extract the granule information from a datapoint given as xarray dataset."""
+        if isinstance(dataset, LocationStorageGranule):
+            return dataset
+
+        if "location" not in dataset:
+            raise ValueError("The given dataset has no location information.")
+
+        thumbnail = None
+        if "thumbnail" in dataset:
+            thumbnail = dataset.thumbnail.item()
+        elif "overview" in dataset:
+            thumbnail = dataset.overview.item()
+        elif "quicklook" in dataset:
+            thumbnail = dataset.quicklook.item()
+
+        return cls(dataset.location.item(), thumbnail)

{tilebox_storage-0.46.0.dist-info → tilebox_storage-0.48.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tilebox-storage
-Version: 0.46.0
+Version: 0.48.0
 Summary: Storage client for Tilebox
 Project-URL: Homepage, https://tilebox.com
 Project-URL: Documentation, https://docs.tilebox.com/
@@ -20,6 +20,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Requires-Python: >=3.10
 Requires-Dist: aiofile>=3.8
+Requires-Dist: boto3>=1.37.0
 Requires-Dist: folium>=0.15
 Requires-Dist: httpx>=0.27
 Requires-Dist: obstore>=0.8.0

tilebox_storage-0.48.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+tilebox/storage/__init__.py,sha256=1oyjSgUcmPpD01mlL_2FSIShRU1uYnZd79UigAwuYxc,3759
+tilebox/storage/aio.py,sha256=mFFWb-WdaaIzRnn0zoRsKWvEhbHNhl7_uEIvUc-9I8E,41818
+tilebox/storage/granule.py,sha256=wdW-TYWA3Ha5R36KGpW0M17vxP-UY7S7cFTTuY1lNSI,7045
+tilebox/storage/providers.py,sha256=vOTxSj2VIQhbFyvxu_eOcPmBGETDaijRoCWi9heUwRs,1832
+tilebox_storage-0.48.0.dist-info/METADATA,sha256=0pu3jMyaUuTVXClSh4f-5pwUfP_-DUgKZG_nsPhcADc,4132
+tilebox_storage-0.48.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tilebox_storage-0.48.0.dist-info/RECORD,,

tilebox_storage-0.46.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-tilebox/storage/__init__.py,sha256=nQYsEKee3lBCDi_rmISGd-kKgqDV75ogiadbpLKLGww,3290
-tilebox/storage/aio.py,sha256=kNahmyUUXeFMgA-XvBXq3MCqBkZw-8BPLr7n2HLf5gA,38383
-tilebox/storage/granule.py,sha256=RPw3UkiIwGwQEqmiuxy2tbWAMrjoMYNNigXimB4jJGI,6179
-tilebox/storage/providers.py,sha256=vOTxSj2VIQhbFyvxu_eOcPmBGETDaijRoCWi9heUwRs,1832
-tilebox_storage-0.46.0.dist-info/METADATA,sha256=IR1nWsAlhWmCZkd0xsnOlHdpubT4xTotW4ukY_YxmyI,4103
-tilebox_storage-0.46.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tilebox_storage-0.46.0.dist-info/RECORD,,