cubexpress 0.1.18__tar.gz → 0.1.21__tar.gz

{cubexpress-0.1.18 → cubexpress-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cubexpress
-Version: 0.1.18
+Version: 0.1.21
 Summary: Efficient processing of cubic Earth-observation (EO) data.
 Home-page: https://github.com/andesdatacube/cubexpress
 Keywords: earth-engine,sentinel-2,geospatial,eo,cube
@@ -16,13 +16,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Scientific/Engineering :: GIS
 Requires-Dist: earthengine-api (>=1.5.12)
-Requires-Dist: numpy (>=2.0.2)
-Requires-Dist: pandas (>=2.2.2)
+Requires-Dist: numpy (>=1.22.4,<2.0)
+Requires-Dist: pandas (>=2.0.0)
 Requires-Dist: pyarrow (>=14.0.0)
-Requires-Dist: pydantic (>=2.11.4)
+Requires-Dist: pydantic (>=2.0.0)
 Requires-Dist: pygeohash (>=1.2.0)
 Requires-Dist: pyproj (>=3.6.0)
 Requires-Dist: rasterio (>=1.3.9)
+Requires-Dist: tqdm (>=4.65.0)
 Requires-Dist: utm (>=0.7.0)
 Project-URL: Documentation, https://andesdatacube.github.io/cubexpress
 Project-URL: Repository, https://github.com/andesdatacube/cubexpress

cubexpress-0.1.21/cubexpress/__init__.py

@@ -0,0 +1,34 @@
+"""
+CubExpress - Efficient Earth Engine data download and processing.
+
+Main components:
+- lonlat2rt: Convert coordinates to raster transforms
+- s2_table: Query Sentinel-2 metadata with cloud scores
+- table_to_requestset: Build request sets from metadata
+- get_cube: Download Earth Engine data cubes
+"""
+
+from __future__ import annotations
+
+from cubexpress.cloud_utils import s2_table
+from cubexpress.conversion import geo2utm, lonlat2rt
+from cubexpress.cube import get_cube
+from cubexpress.geotyping import RasterTransform, Request, RequestSet
+from cubexpress.request import table_to_requestset
+
+__all__ = [
+    "lonlat2rt",
+    "geo2utm",
+    "RasterTransform",
+    "Request",
+    "RequestSet",
+    "s2_table",
+    "table_to_requestset",
+    "get_cube",
+]
+
+try:
+    from importlib.metadata import version
+    __version__ = version("cubexpress")
+except Exception:
+    __version__ = "0.0.0-dev"

cubexpress-0.1.21/cubexpress/cache.py

@@ -0,0 +1,76 @@
+"""Caching utilities for Earth Engine query results."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import pathlib
+
+from cubexpress.config import CACHE_DIR
+
+CACHE_DIR.mkdir(exist_ok=True, parents=True)
+
+
+def _cache_key(
+    lon: float,
+    lat: float,
+    edge_size: int | tuple[int, int],
+    scale: int,
+    collection: str,
+) -> pathlib.Path:
+    """
+    Generate a deterministic cache file path for query parameters.
+
+    Coordinates are rounded to 4 decimal places (~11m precision) to
+    ensure cache hits for equivalent locations.
+
+    Args:
+        lon: Longitude of center point
+        lat: Latitude of center point
+        edge_size: ROI size in pixels
+        scale: Pixel resolution in meters
+        collection: Earth Engine collection ID
+
+    Returns:
+        Path to hashed .parquet cache file
+    """
+    lon_r = round(lon, 4)
+    lat_r = round(lat, 4)
+    
+    edge_tuple = (
+        (edge_size, edge_size) if isinstance(edge_size, int) 
+        else tuple(edge_size)
+    )
+    
+    signature = [lon_r, lat_r, edge_tuple, scale, collection]
+    
+    raw = json.dumps(signature, sort_keys=True).encode("utf-8")
+    digest = hashlib.md5(raw).hexdigest()
+    
+    return CACHE_DIR / f"{digest}.parquet"
+
+
+def clear_cache() -> int:
+    """
+    Remove all cached query results.
+    
+    Returns:
+        Number of files deleted
+    """
+    count = 0
+    for cache_file in CACHE_DIR.glob("*.parquet"):
+        cache_file.unlink()
+        count += 1
+    return count
+
+
+def get_cache_size() -> tuple[int, int]:
+    """
+    Calculate total cache size.
+    
+    Returns:
+        Tuple of (file_count, total_bytes)
+    """
+    files = list(CACHE_DIR.glob("*.parquet"))
+    total_bytes = sum(f.stat().st_size for f in files)
+    return len(files), total_bytes

{cubexpress-0.1.18 → cubexpress-0.1.21}/cubexpress/cloud_utils.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
 import datetime as dt
-import sys
 import time
+import warnings
+
 import ee
 import pandas as pd
+
 from cubexpress.cache import _cache_key
-import datetime as dt
 from cubexpress.geospatial import _square_roi
-import warnings
+
 warnings.filterwarnings('ignore', category=DeprecationWarning)
 
 
@@ -241,7 +242,9 @@ def s2_table(
         elapsed = time.time() - t0
         n_images = len(df_full)
         date_range = f"{start} to {end}"
-        print(f"\r✅ Retrieved {n_images} images from {date_range} ({elapsed:.2f}s)")
+        actual_start = df_full['date'].min()
+        actual_end = df_full['date'].max()
+        print(f"\r✅ Retrieved {n_images} images from {actual_start} to {actual_end} ({elapsed:.2f}s)")
 
     # Save cache
     if cache:

cubexpress-0.1.21/cubexpress/conversion.py

@@ -0,0 +1,147 @@
+"""Coordinate conversion and raster transform utilities."""
+
+from __future__ import annotations
+
+import utm
+from pyproj import CRS, Transformer
+
+from cubexpress.exceptions import ValidationError
+from cubexpress.geotyping import RasterTransform
+
+
+def parse_edge_size(edge_size: int | tuple[int, int]) -> tuple[int, int]:
+    """
+    Parse edge_size input into (width, height) tuple.
+    
+    Args:
+        edge_size: Size specification (int for square, tuple for rectangle)
+        
+    Returns:
+        Tuple of (width, height) in pixels
+        
+    Raises:
+        ValidationError: If input is invalid
+    """
+    if isinstance(edge_size, int):
+        if edge_size <= 0:
+            raise ValidationError(f"edge_size must be positive, got {edge_size}")
+        return (edge_size, edge_size)
+    
+    if len(edge_size) != 2:
+        raise ValidationError(
+            f"edge_size tuple must have 2 elements, got {len(edge_size)}"
+        )
+    
+    width, height = edge_size
+    if width <= 0 or height <= 0:
+        raise ValidationError(
+            f"edge_size values must be positive, got {edge_size}"
+        )
+    
+    return (width, height)
+
+
+def geo2utm(lon: float, lat: float) -> tuple[float, float, str]:
+    """
+    Convert lat/lon to UTM coordinates and EPSG code.
+    
+    Uses the utm library for standard conversion.
+    
+    Args:
+        lon: Longitude in decimal degrees
+        lat: Latitude in decimal degrees
+
+    Returns:
+        Tuple of (x, y, epsg_code) where EPSG code is formatted as 'EPSG:XXXXX'
+        
+    Raises:
+        utm.OutOfRangeError: If coordinates are outside valid UTM range
+    """
+    x, y, zone, _ = utm.from_latlon(lat, lon)
+    epsg_code = f"326{zone:02d}" if lat >= 0 else f"327{zone:02d}"
+    return float(x), float(y), f"EPSG:{epsg_code}"
+
+
+def lonlat2rt_utm_or_ups(lon: float, lat: float) -> tuple[float, float, str]:
+    """
+    Calculate UTM coordinates using pyproj (fallback for geo2utm).
+    
+    This method is more robust than the utm library and works globally,
+    including near the poles. Uses standard UTM zones for all latitudes
+    to match Google Earth Engine behavior.
+    
+    Args:
+        lon: Longitude in decimal degrees
+        lat: Latitude in decimal degrees
+        
+    Returns:
+        Tuple of (x, y, epsg_code)
+    """
+    zone = int((lon + 180) // 6) + 1
+    epsg_code = 32600 + zone if lat >= 0 else 32700 + zone
+    crs = CRS.from_epsg(epsg_code)
+    
+    transformer = Transformer.from_crs(4326, crs, always_xy=True)
+    x, y = transformer.transform(lon, lat)
+    
+    return float(x), float(y), f"EPSG:{epsg_code}"
+
+
+def lonlat2rt(
+    lon: float, 
+    lat: float, 
+    edge_size: int | tuple[int, int], 
+    scale: int
+) -> RasterTransform:
+    """
+    Generate a RasterTransform from geographic coordinates.
+
+    Converts (lon, lat) to UTM projection and builds geospatial metadata
+    including affine transformation parameters. The Y-scale is negative
+    because raster images have their origin at the top-left corner.
+
+    Args:
+        lon: Longitude in decimal degrees
+        lat: Latitude in decimal degrees
+        edge_size: Output raster size
+            - int: creates square (width=height=edge_size)
+            - tuple: specifies (width, height) in pixels
+        scale: Spatial resolution in meters per pixel
+
+    Returns:
+        RasterTransform with CRS, geotransform, and dimensions
+
+    Examples:
+        >>> rt = lonlat2rt(lon=-76.0, lat=40.0, edge_size=512, scale=30)
+        >>> print(rt.width, rt.height)
+        512 512
+        
+        >>> rt = lonlat2rt(lon=-76.0, lat=40.0, edge_size=(1024, 512), scale=30)
+        >>> print(rt.width, rt.height)
+        1024 512
+    """
+    try:
+        x, y, crs = geo2utm(lon, lat)
+    except Exception:
+        x, y, crs = lonlat2rt_utm_or_ups(lon, lat)
+    
+    width, height = parse_edge_size(edge_size)
+    
+    half_width_m = (width * scale) / 2
+    half_height_m = (height * scale) / 2
+
+    geotransform = {
+        "scaleX": scale,
+        "shearX": 0,
+        "translateX": x - half_width_m,
+        "scaleY": -scale,
+        "shearY": 0,
+        "translateY": y + half_height_m,
+    }
+
+    return RasterTransform(
+        crs=crs,
+        geotransform=geotransform,
+        width=width,
+        height=height
+    )

cubexpress-0.1.21/cubexpress/cube.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import json
+import pathlib
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from copy import deepcopy
+from typing import Any
+
+import ee
+import pandas as pd
+from tqdm import tqdm
+
+from cubexpress.downloader import download_manifest, download_manifests
+from cubexpress.geospatial import calculate_cell_size, quadsplit_manifest
+from cubexpress.geotyping import RequestSet
+from cubexpress.logging_config import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def _test_manifest_tiling(manifest: dict[str, Any]) -> int:
+    """
+    Test if a manifest requires tiling without downloading data.
+    
+    Args:
+        manifest: Earth Engine download manifest
+        
+    Returns:
+        Number of tiles required (1 if no tiling needed)
+    """
+    try:
+        if "assetId" in manifest:
+            _ = ee.data.getPixels(manifest)
+        elif "expression" in manifest:
+            ee_image = ee.deserializer.decode(json.loads(manifest["expression"]))
+            manifest_copy = deepcopy(manifest)
+            manifest_copy["expression"] = ee_image
+            _ = ee.data.computePixels(manifest_copy)
+        
+        return 1
+        
+    except ee.ee_exception.EEException as err:
+        size = manifest["grid"]["dimensions"]["width"]
+        cell_w, cell_h, power = calculate_cell_size(str(err), size)
+        n_tiles = (2 ** power) ** 2
+        return n_tiles
+
+
+def get_geotiff(
+    manifest: dict[str, Any],
+    full_outname: pathlib.Path | str,
+    nworks: int,
+    return_tile_info: bool = False,
+) -> int | None:
+    """
+    Download a single GeoTIFF with automatic tiling if needed.
+
+    Args:
+        manifest: Earth Engine download manifest
+        full_outname: Output path for final GeoTIFF
+        nworks: Number of worker threads for tiling
+        return_tile_info: If True, return number of tiles created
+
+    Returns:
+        Number of tiles if return_tile_info=True, otherwise None
+    """
+    try:
+        download_manifest(ulist=manifest, full_outname=full_outname)
+        return 1 if return_tile_info else None
+        
+    except ee.ee_exception.EEException as err:
+        size = manifest["grid"]["dimensions"]["width"]
+        cell_w, cell_h, power = calculate_cell_size(str(err), size)
+        
+        tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
+        n_tiles = len(tiled)
+        
+        # Silent tiling - no log spam
+        download_manifests(
+            manifests=tiled,
+            full_outname=full_outname,
+            max_workers=nworks
+        )
+        
+        return n_tiles if return_tile_info else None
+
+
+def _detect_optimal_workers(
+    first_manifest: dict[str, Any],
+    total_workers: int
+) -> tuple[int, int]:
+    """
+    Detect optimal worker distribution by testing first image.
+    
+    Args:
+        first_manifest: Manifest of first image
+        total_workers: Total workers to distribute
+        
+    Returns:
+        Tuple of (outer_workers, inner_workers)
+    """
+    n_tiles = _test_manifest_tiling(first_manifest)
+    
+    if n_tiles == 1:
+        outer, inner = total_workers, 1
+        logger.debug(f"No tiling needed - using {outer} parallel images")
+    else:
+        inner = min(n_tiles, max(1, total_workers // 2))
+        outer = max(1, total_workers // inner)
+        logger.info(
+            f"Auto-detected tiling required ({n_tiles} tiles/image) - "
+            f"using outer={outer}, inner={inner}"
+        )
+    
+    return outer, inner
+
+
+def get_cube(
+    requests: pd.DataFrame | RequestSet,
+    outfolder: pathlib.Path | str,
+    nworks: int | tuple[int, int] = 4,
+    auto_workers: bool = True
+) -> None:
+    """
+    Download a set of Earth Engine requests in parallel.
+
+    Args:
+        requests: Collection of requests (DataFrame or RequestSet)
+        outfolder: Destination directory
+        nworks: Worker configuration (int or tuple of (outer, inner))
+        auto_workers: If True, automatically detect optimal distribution
+
+    Raises:
+        ValueError: If nworks configuration is invalid
+        TypeError: If nworks has wrong type
+    """
+    outfolder = pathlib.Path(outfolder).expanduser().resolve()
+    outfolder.mkdir(parents=True, exist_ok=True)
+    
+    dataframe = (
+        requests._dataframe if isinstance(requests, RequestSet) 
+        else requests
+    )
+    
+    if dataframe.empty:
+        logger.warning("Request set is empty")
+        return
+    
+    # Determine worker configuration
+    if isinstance(nworks, int):
+        if nworks <= 0:
+            raise ValueError(f"nworks must be positive, got {nworks}")
+        
+        if auto_workers:
+            first_row = dataframe.iloc[0]
+            nworks_outer, nworks_inner = _detect_optimal_workers(
+                first_manifest=first_row.manifest,
+                total_workers=nworks
+            )
+        else:
+            nworks_outer, nworks_inner = nworks, 1
+            
+    elif isinstance(nworks, (list, tuple)):
+        if len(nworks) != 2:
+            raise ValueError(f"nworks tuple must have 2 elements, got {len(nworks)}")
+        
+        nworks_outer, nworks_inner = nworks
+        
+        if not all(isinstance(n, int) for n in (nworks_outer, nworks_inner)):
+            raise TypeError(f"nworks elements must be integers")
+        
+        if nworks_outer <= 0 or nworks_inner <= 0:
+            raise ValueError(f"nworks values must be positive")
+    else:
+        raise TypeError(f"nworks must be int or tuple, got {type(nworks)}")
+    
+    # Execute downloads
+    failed = []
+    with ThreadPoolExecutor(max_workers=nworks_outer) as executor:
+        futures = {
+            executor.submit(
+                get_geotiff,
+                manifest=row.manifest,
+                full_outname=outfolder / f"{row.id}.tif",
+                nworks=nworks_inner,
+                return_tile_info=False
+            ): row.id 
+            for _, row in dataframe.iterrows()
+        }
+
+        for future in tqdm(
+            as_completed(futures), 
+            total=len(futures),
+            desc=f"Downloading (outer={nworks_outer}, inner={nworks_inner})",
+            unit="image",
+            leave=True
+        ):
+            img_id = futures[future]
+            try:
+                future.result()
+            except Exception as exc:
+                logger.error(f"Failed {img_id}: {exc}")
+                failed.append(img_id)
+    
+    # Summary
+    if failed:
+        logger.warning(f"{len(failed)}/{len(dataframe)} downloads failed")
+    else:
+        logger.info(f"✓ Downloaded {len(dataframe)} images to {outfolder}")

cubexpress-0.1.21/cubexpress/downloader.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+import pathlib
+import shutil
+import tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import contextmanager
+from copy import deepcopy
+from typing import Any, Iterator
+
+import ee
+
+from cubexpress.geospatial import merge_tifs
+
+
+@contextmanager
+def temp_workspace(prefix: str = "cubexpress_") -> Iterator[pathlib.Path]:
+    """
+    Create a temporary directory with automatic cleanup.
+    
+    Args:
+        prefix: Prefix for the temporary directory name
+        
+    Yields:
+        Path to temporary directory
+    """
+    tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix=prefix))
+    try:
+        yield tmp_dir
+    finally:
+        if tmp_dir.exists():
+            shutil.rmtree(tmp_dir, ignore_errors=True)
+
+
+def download_manifest(
+    ulist: dict[str, Any], 
+    full_outname: pathlib.Path
+) -> None:
+    """
+    Download data from Earth Engine based on a manifest dictionary.
+
+    Handles both direct asset IDs and serialized EE expressions.
+
+    Args:
+        ulist: Export manifest containing 'assetId' or 'expression'
+        full_outname: Destination path for the downloaded file
+
+    Raises:
+        ValueError: If manifest is invalid
+        ee.ee_exception.EEException: If Earth Engine request fails
+    """
+    if "assetId" in ulist:
+        images_bytes = ee.data.getPixels(ulist)
+    elif "expression" in ulist:
+        ee_image = ee.deserializer.decode(json.loads(ulist["expression"]))
+        ulist_deep = deepcopy(ulist)
+        ulist_deep["expression"] = ee_image
+        images_bytes = ee.data.computePixels(ulist_deep)
+    else:
+        raise ValueError("Manifest must contain 'assetId' or 'expression'")
+    
+    full_outname.parent.mkdir(parents=True, exist_ok=True)
+    with open(full_outname, "wb") as f:
+        f.write(images_bytes)
+
+
+def download_manifests(
+    manifests: list[dict[str, Any]],
+    full_outname: pathlib.Path,
+    max_workers: int = 1,
+) -> None:
+    """
+    Download multiple manifests concurrently and merge into one file.
+
+    Uses a temporary workspace that is automatically cleaned up.
+
+    Args:
+        manifests: List of Earth Engine manifests
+        full_outname: Final destination path for merged TIFF
+        max_workers: Number of parallel download threads
+
+    Raises:
+        ee.ee_exception.EEException: If any download fails
+        ValueError: If merge fails
+    """
+    with temp_workspace() as tmp_dir:
+        tile_dir = tmp_dir / full_outname.stem
+        tile_dir.mkdir(parents=True, exist_ok=True)
+
+        # Download tiles in parallel
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(
+                    download_manifest, 
+                    ulist=manifest, 
+                    full_outname=tile_dir / f"{idx:06d}.tif"
+                ): idx 
+                for idx, manifest in enumerate(manifests)
+            }
+            
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as exc:
+                    idx = futures[future]
+                    print(f"Error downloading tile {idx}: {exc}")
+                    raise
+        
+        # Merge tiles
+        input_files = sorted(tile_dir.glob("*.tif"))
+        if not input_files:
+            raise ValueError(f"No tiles downloaded in {tile_dir}")
+        
+        merge_tifs(input_files, full_outname)