cubexpress 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

cubexpress/__init__.py

@@ -1,15 +1,22 @@
-from cubexpress.conversion import lonlat2rt
-from cubexpress.download import getcube, getGeoTIFF
+from cubexpress.conversion import lonlat2rt, geo2utm
 from cubexpress.geotyping import RasterTransform, Request, RequestSet
+from cubexpress.cloud_utils import cloud_table
+from cubexpress.cube import get_cube
+from cubexpress.request import table_to_requestset
 
+
+
+# pyproj
 # Export the functions
 __all__ = [
     "lonlat2rt",
     "RasterTransform",
     "Request",
     "RequestSet",
-    "getcube",
-    "getGeoTIFF",
+    "geo2utm",
+    "get_cube",
+    "cloud_table",
+    "table_to_requestset"
 ]
 
 # Dynamic version import

cubexpress/cache.py

@@ -0,0 +1,50 @@
+"""Simple file-based cache helpers for cloud_table results."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import pathlib
+from typing import Final
+
+# Folder where per-location parquet files are stored.
+_CACHE_DIR: Final[pathlib.Path] = pathlib.Path(
+    os.getenv("CUBEXPRESS_CACHE", "~/.cubexpress_cache")
+).expanduser()
+_CACHE_DIR.mkdir(exist_ok=True)
+
+
+def _cache_key(
+    lon: float,
+    lat: float,
+    edge_size: int,
+    scale: int,
+    collection: str,
+) -> pathlib.Path:
+    """Return deterministic parquet path for the given query parameters.
+
+    A 128-bit MD5 hash of the rounded coordinates, edge size, scale and
+    collection is used as file name to avoid overly long paths and ensure
+    uniqueness.
+
+    Parameters
+    ----------
+    lon, lat
+        Centre coordinates in decimal degrees; rounded to 4 dp (≈ 11 m).
+    edge_size
+        Edge length in pixels of the requested square ROI.
+    scale
+        Pixel size in metres.
+    collection
+        EE collection name (e.g. ``"COPERNICUS/S2_HARMONIZED"``).
+
+    Returns
+    -------
+    pathlib.Path
+        Absolute path ending in ``.parquet`` under ``_CACHE_DIR``.
+    """
+    lon_r, lat_r = round(lon, 4), round(lat, 4)
+    raw = json.dumps([lon_r, lat_r, edge_size, scale, collection]).encode()
+    digest = hashlib.md5(raw).hexdigest()  # noqa: S324 – non-cryptographic OK
+    return _CACHE_DIR / f"{digest}.parquet"

cubexpress/cloud_utils.py

@@ -0,0 +1,256 @@
+"""Cloud-coverage tables for Sentinel-2 over a square ROI.
+
+Two helpers are exposed:
+
+* :func:`_cloud_table_single_range` – query Earth Engine for one date-range.
+* :func:`cloud_table` – smart wrapper that adds on-disk caching, automatic
+  back-filling, and cloud-percentage filtering.
+
+Both return a ``pandas.DataFrame`` with the columns **day**, **cloudPct** and
+**images** plus useful ``.attrs`` metadata for downstream functions.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import pathlib
+from typing import List, Optional
+
+import ee
+import pandas as pd
+
+from cubexpress.cache import _cache_key
+from cubexpress.geospatial import _square_roi
+
+
+def _cloud_table_single_range(
+    lon: float,
+    lat: float,
+    edge_size: int,
+    scale: int,
+    start: str,
+    end: str,
+    collection: str = "COPERNICUS/S2_HARMONIZED",
+) -> pd.DataFrame:
+    """Return raw cloud-table rows for a single *start–end* interval.
+
+    Parameters
+    ----------
+    lon, lat
+        Centre coordinates in decimal degrees.
+    edge_size, scale
+        ROI size in pixels (*edge_size*) and pixel resolution in metres
+        (*scale*), fed into :pyfunc:`cubexpress.geospatial._square_roi`.
+    start, end
+        ISO-dates (``YYYY-MM-DD``) delimiting the query.
+    collection
+        Sentinel-2 collection name to query.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Columns: **day** (str), **cloudPct** (float), **images** (str
+        concatenation of asset IDs separated by ``-``). No filtering applied.
+    """
+    roi = _square_roi(lon, lat, edge_size, scale)
+    s2 = ee.ImageCollection(collection)
+
+    if collection in (
+        "COPERNICUS/S2_HARMONIZED",
+        "COPERNICUS/S2_SR_HARMONIZED",
+    ):
+        qa_band = "cs_cdf"
+        csp = ee.ImageCollection("GOOGLE/CLOUD_SCORE_PLUS/V1/S2_HARMONIZED")
+    else:
+        qa_band, csp = None, None
+
+    def _add_props(img):
+        day = ee.Date(img.get("system:time_start")).format("YYYY-MM-dd")
+        imgid = img.get("system:index")
+
+        if qa_band:
+            score = (
+                img.linkCollection(csp, [qa_band])
+                .select([qa_band])
+                .reduceRegion(ee.Reducer.mean(), roi, scale)
+                .get(qa_band)
+            )
+            # If score is null assume completely clear (score=1 → cloudPct=0)
+            score_safe = ee.Algorithms.If(score, score, -1)
+            cloud_pct = (
+                ee.Number(1)
+                .subtract(ee.Number(score_safe))
+                .multiply(10000)
+                .round()
+                .divide(100)
+            )
+        else:
+            cloud_pct = ee.Number(-1)
+
+        return ee.Feature(
+            None,
+            {
+                "day": day,
+                "cloudPct": cloud_pct,
+                "images": imgid,
+            },
+        )
+
+    triples = (
+        s2.filterDate(start, end)
+        .filterBounds(roi)
+        .map(_add_props)
+        .reduceColumns(ee.Reducer.toList(3), ["day", "cloudPct", "images"])
+        .get("list")
+        .getInfo()
+    )
+
+    df = pd.DataFrame(triples, columns=["day", "cloudPct", "images"]).dropna()
+    df["cloudPct"] = df["cloudPct"].astype(float)
+    df["images"] = df["images"].astype(str)
+    return df
+
+
+def cloud_table(
+    lon: float,
+    lat: float,
+    edge_size: int = 2048,
+    scale: int = 10,
+    start: str = "2017-01-01",
+    end: str = "2024-12-31",
+    cloud_max: float = 7.0,
+    bands: Optional[List[str]] = None,
+    collection: str = "COPERNICUS/S2_HARMONIZED",
+    output_path: str | pathlib.Path | None = None,
+    cache: bool = True,
+    verbose: bool = True,
+) -> pd.DataFrame:
+    """Build (and cache) a per-day cloud-table for the requested ROI.
+
+    The function first checks an on-disk parquet cache keyed on location and
+    parameters.  If parts of the requested date-range are missing, it fetches
+    only those gaps from Earth Engine, merges, updates the cache and finally
+    filters by *cloud_max*.
+
+    Parameters
+    ----------
+    lon, lat
+        Centre coordinates.
+    edge_size, scale
+        Square size (pixels) and resolution (metres).
+    start, end
+        ISO start/end dates.
+    cloud_max
+        Maximum allowed cloud percentage (0-100). Rows above this threshold are
+        dropped.
+    bands
+        List of spectral bands to embed as metadata.  If *None* the full
+        Sentinel-2 set is used.
+    collection
+        Sentinel-2 collection to query.
+    output_path
+        Downstream path hint stored in ``result.attrs``; not used internally.
+    cache
+        Toggle parquet caching.
+    verbose
+        If *True* prints cache info/progress.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Filtered cloud table with ``.attrs`` containing the call parameters.
+    """
+    if bands is None:
+        bands = [
+            "B1",
+            "B2",
+            "B3",
+            "B4",
+            "B5",
+            "B6",
+            "B7",
+            "B8",
+            "B8A",
+            "B9",
+            "B10",
+            "B11",
+            "B12",
+        ]
+
+    cache_file = _cache_key(lon, lat, edge_size, scale, collection)
+
+    # ─── 1. Load cached data if present ────────────────────────────────────
+    if cache and cache_file.exists():
+        if verbose:
+            print("📂  Loading cached table …")
+        df_cached = pd.read_parquet(cache_file)
+        have_idx = pd.to_datetime(df_cached["day"], errors="coerce").dropna()
+
+        cached_start = have_idx.min().date()
+        cached_end = have_idx.max().date()
+
+        if (
+            dt.date.fromisoformat(start) >= cached_start
+            and dt.date.fromisoformat(end) <= cached_end
+        ):
+            if verbose:
+                print("✅  Served entirely from cache.")
+            df_full = df_cached
+        else:
+            # Identify missing segments and fetch only those.
+            df_new_parts = []
+            if dt.date.fromisoformat(start) < cached_start:
+                a1, b1 = start, cached_start.isoformat()
+                df_new_parts.append(
+                    _cloud_table_single_range(
+                        lon, lat, edge_size, scale, a1, b1, collection
+                    )
+                )
+            if dt.date.fromisoformat(end) > cached_end:
+                a2, b2 = cached_end.isoformat(), end
+                df_new_parts.append(
+                    _cloud_table_single_range(
+                        lon, lat, edge_size, scale, a2, b2, collection
+                    )
+                )
+            df_new = pd.concat(df_new_parts, ignore_index=True)
+            df_full = (
+                pd.concat([df_cached, df_new], ignore_index=True)
+                .drop_duplicates("day")
+                .sort_values("day", kind="mergesort")
+            )
+    else:
+        # No cache or caching disabled: fetch full range.
+        if verbose:
+            msg = "Generating table (no cache found)…" if cache else "Generating table…"
+            print("⏳", msg)
+        df_full = _cloud_table_single_range(
+            lon, lat, edge_size, scale, start, end, collection
+        )
+
+    # ─── 2. Save cache ─────────────────────────────────────────────────────
+    if cache:
+        df_full.to_parquet(cache_file, compression="zstd")
+
+    # ─── 3. Filter by cloud cover and requested date window ────────────────
+    result = (
+        df_full.query("@start <= day <= @end")
+        .query("cloudPct < @cloud_max")
+        .reset_index(drop=True)
+    )
+
+    # Attach metadata for downstream helpers
+    result.attrs.update(
+        {
+            "lon": lon,
+            "lat": lat,
+            "edge_size": edge_size,
+            "scale": scale,
+            "bands": bands,
+            "collection": collection,
+            "cloud_max": cloud_max,
+            "output_path": str(output_path) if output_path else "",
+        }
+    )
+    return result

cubexpress/conversion.py

@@ -19,7 +19,7 @@ def geo2utm(lon: float, lat: float) -> tuple[float, float, str]:
     """
     x, y, zone, _ = utm.from_latlon(lat, lon)
     epsg_code = f"326{zone:02d}" if lat >= 0 else f"327{zone:02d}"
-    return x, y, f"EPSG:{epsg_code}"
+    return float(x), float(y), f"EPSG:{epsg_code}"
 
 
 def lonlat2rt(lon: float, lat: float, edge_size: int, scale: int) -> RasterTransform:

cubexpress/cube.py

@@ -0,0 +1,84 @@
+"""High-level helpers for tiled GeoTIFF downloads.
+
+The module provides two thread-friendly wrappers:
+
+* **get_geotiff** – download a single manifest, auto-tiling on EE pixel-count
+  errors.
+* **get_cube** – iterate over a ``RequestSet`` (or similar) and build a local
+  raster “cube” in parallel.
+
+The core download/split logic lives in *cubexpress.downloader* and
+*cubexpress.geospatial*; here we merely orchestrate it.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import concurrent.futures
+from typing import Dict, Any
+
+import ee
+from cubexpress.downloader import download_manifest, download_manifests
+from cubexpress.geospatial import quadsplit_manifest, calculate_cell_size
+from cubexpress.geotyping import RequestSet
+
+
+def get_geotiff(
+    manifest: Dict[str, Any],
+    full_outname: pathlib.Path | str,
+    nworks: int = 4,
+) -> None:
+    """Download *manifest* to *full_outname*, retrying with tiled requests.
+
+    Parameters
+    ----------
+    manifest
+        Earth Engine download manifest returned by cubexpress.
+    full_outname
+        Final ``.tif`` path (created/overwritten).
+    nworks
+        Maximum worker threads when the image must be split; default **4**.
+    """
+    try:
+        download_manifest(manifest, pathlib.Path(full_outname))
+    except ee.ee_exception.EEException as err:
+        # Handle EE “too many pixels” error by recursive tiling.
+        size = manifest["grid"]["dimensions"]["width"]  # square images assumed
+        cell_w, cell_h, power = calculate_cell_size(str(err), size)
+        tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
+        download_manifests(tiled, max_workers=nworks, full_outname=pathlib.Path(full_outname))
+
+
+def get_cube(
+    requests: RequestSet,
+    outfolder: pathlib.Path | str,
+    nworks: int = 4,
+) -> None:
+    """Download every request in *requests* to *outfolder* using a thread pool.
+
+    Each row in ``requests._dataframe`` must expose ``manifest`` and ``id``.
+    Resulting files are named ``{id}.tif``.
+
+    Parameters
+    ----------
+    requests
+        A ``RequestSet`` or object with an internal ``_dataframe`` attribute.
+    outfolder
+        Folder where the GeoTIFFs will be written (created if absent).
+    nworks
+        Pool size for concurrent downloads; default **4**.
+    """
+    out = pathlib.Path(outfolder)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=nworks) as pool:
+        futures = []
+        for _, row in requests._dataframe.iterrows():
+            outname = out / f"{row.id}.tif"
+            outname.parent.mkdir(parents=True, exist_ok=True)
+            futures.append(pool.submit(get_geotiff, row.manifest, outname, nworks))
+
+        for fut in concurrent.futures.as_completed(futures):
+            try:
+                fut.result()
+            except Exception as exc:  # noqa: BLE001 – log and keep going
+                print(f"Download error: {exc}")

cubexpress/downloader.py

@@ -0,0 +1,95 @@
+"""Low-level download helpers for Earth Engine manifests.
+
+Only two public callables are exposed:
+
+* :func:`download_manifest` – fetch a single manifest and write one GeoTIFF.
+* :func:`download_manifests` – convenience wrapper to parallel-download a list
+  of manifests with a thread pool.
+
+Both functions are fully I/O bound; no return value is expected.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import concurrent.futures
+from copy import deepcopy
+from typing import Any, Dict, List
+
+import ee
+import rasterio as rio
+from rasterio.io import MemoryFile
+
+
+def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None:
+    """Download *ulist* and save it as *full_outname*.
+
+    The manifest must include either an ``assetId`` or an ``expression``
+    (serialized EE image). RasterIO is used to write a tiled, compressed
+    GeoTIFF; the function is silent apart from the final ``print``.
+    """
+    if "assetId" in ulist:
+        images_bytes = ee.data.getPixels(ulist)
+    elif "expression" in ulist:
+        ee_image = ee.deserializer.decode(json.loads(ulist["expression"]))
+        ulist_deep = deepcopy(ulist)
+        ulist_deep["expression"] = ee_image
+        images_bytes = ee.data.computePixels(ulist_deep)
+    else:  # pragma: no cover
+        raise ValueError("Manifest does not contain 'assetId' or 'expression'")
+
+    with MemoryFile(images_bytes) as memfile:
+        with memfile.open() as src:
+            profile = src.profile
+            profile.update(
+                {
+                    "driver": "Gtiff",
+                    "tiled": "yes",
+                    "interleave": "band",
+                    "blockxsize": 256,
+                    "blockysize": 256,
+                    "compress": "ZSTD",
+                    "predictor": 2,
+                    "num_threads": 20,
+                    "nodata": 65535,
+                    "dtype": "uint16",
+                    "count": 13,
+                    "lztd_level": 13,
+                    "copy_src_overviews": True,
+                    "overviews": "AUTO",
+                }
+            )
+            all_bands = src.read()
+
+    with rio.open(full_outname, "w", **profile) as dst:
+        dst.write(all_bands)
+
+    print(f"{full_outname} downloaded successfully.")  # noqa: T201
+
+
+def download_manifests(
+    manifests: List[Dict[str, Any]],
+    max_workers: int,
+    full_outname: pathlib.Path,
+) -> None:
+    """Download every manifest in *manifests* concurrently.
+
+    Each output file is saved in the folder
+    ``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
+    ``000001.tif`` … according to the list order.
+    """
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = []
+
+        for index, umanifest in enumerate(manifests):
+            folder = full_outname.parent / full_outname.stem
+            folder.mkdir(parents=True, exist_ok=True)
+            outname = folder / f"{index:06d}.tif"
+            futures.append(executor.submit(download_manifest, umanifest, outname))
+
+        for fut in concurrent.futures.as_completed(futures):
+            try:
+                fut.result()
+            except Exception as exc:  # noqa: BLE001
+                print(f"Error en una de las descargas: {exc}")  # noqa: T201

cubexpress/geospatial.py

@@ -0,0 +1,55 @@
+import ee
+import re
+from copy import deepcopy
+from typing import Dict
+
+
+def quadsplit_manifest(manifest: Dict, cell_width: int, cell_height: int, power: int) -> list[Dict]:
+    manifest_copy = deepcopy(manifest)
+    
+    manifest_copy["grid"]["dimensions"]["width"] = cell_width
+    manifest_copy["grid"]["dimensions"]["height"] = cell_height
+    x = manifest_copy["grid"]["affineTransform"]["translateX"]
+    y = manifest_copy["grid"]["affineTransform"]["translateY"]
+    scale_x = manifest_copy["grid"]["affineTransform"]["scaleX"]
+    scale_y = manifest_copy["grid"]["affineTransform"]["scaleY"]
+
+    manifests = []
+
+    for columny in range(2**power):
+        for rowx in range(2**power):
+            new_x = x + (rowx * cell_width) * scale_x
+            new_y = y + (columny * cell_height) * scale_y
+            new_manifest = deepcopy(manifest_copy)
+            new_manifest["grid"]["affineTransform"]["translateX"] = new_x
+            new_manifest["grid"]["affineTransform"]["translateY"] = new_y
+            manifests.append(new_manifest)
+
+    return manifests
+
+
+
+def calculate_cell_size(ee_error_message: str, size: int) -> tuple[int, int]:
+    match = re.findall(r'\d+', ee_error_message)
+    image_pixel = int(match[0])
+    max_pixel = int(match[1])
+    
+    images = image_pixel / max_pixel
+    power = 0
+    
+    while images > 1:
+        power += 1
+        images = image_pixel / (max_pixel * 4 ** power)
+    
+    cell_width = size // 2 ** power
+    cell_height = size // 2 ** power
+    
+    return cell_width, cell_height, power
+
+
+
+def _square_roi(lon: float, lat: float, edge_size: int, scale: int) -> ee.Geometry:
+    """Return a square `ee.Geometry` centred on (*lon*, *lat*)."""
+    half = edge_size * scale / 2
+    point = ee.Geometry.Point([lon, lat])
+    return point.buffer(half).bounds()

cubexpress/geotyping.py

@@ -282,7 +282,7 @@ class RequestSet(BaseModel):
         return pd.DataFrame(
             [
                 {
-                    "id": meta.id,
+                    "id": meta.id, # add clud
                     "lon": lon[index],
                     "lat": lat[index],
                     "x": x[index],

cubexpress/request.py

@@ -0,0 +1,77 @@
+"""Convert cloud_table output into a RequestSet."""
+
+from __future__ import annotations
+
+import ee
+import pandas as pd
+import pygeohash as pgh
+from typing import List
+
+from cubexpress.geotyping import Request, RequestSet
+from cubexpress.conversion import lonlat2rt
+
+
+def table_to_requestset(df: pd.DataFrame, *, mosaic: bool = True) -> RequestSet:
+    """Return a :class:`RequestSet` built from *df* (cloud_table result).
+
+    Parameters
+    ----------
+    df
+        DataFrame with *day* and *images* columns plus attrs created by
+        :pyfunc:`cubexpress.cloud_table`.
+    mosaic
+        If ``True`` a single mosaic per day is requested; otherwise each
+        individual asset becomes its own request.
+
+    Raises
+    ------
+    ValueError
+        If *df* is empty after filtering.
+
+    """
+    if df.empty:
+        raise ValueError("cloud_table returned no rows; nothing to request.")
+
+    rt = lonlat2rt(
+        lon=df.attrs["lon"],
+        lat=df.attrs["lat"],
+        edge_size=df.attrs["edge_size"],
+        scale=df.attrs["scale"],
+    )
+    centre_hash = pgh.encode(df.attrs["lat"], df.attrs["lon"], precision=5)
+    reqs: List[Request] = []
+
+    if mosaic:
+        # group all asset IDs per day
+        grouped = (
+            df.assign(img=lambda x: x.images.str.split("-"))
+              .explode("img")
+              .groupby("day")["img"]
+              .apply(list)
+        )
+
+        for day, img_ids in grouped.items():
+            ee_img = ee.ImageCollection(
+                [ee.Image(f"{df.attrs['collection']}/{img}") for img in img_ids]
+            ).mosaic()
+            reqs.append(
+                Request(
+                    id=f"{day}_{centre_hash}_mosaic",
+                    raster_transform=rt,
+                    image=ee_img,
+                    bands=df.attrs["bands"],
+                )
+            )
+    else:  # one request per asset
+        for _, row in df.iterrows():
+            for img_id in row["images"].split("-"):
+                reqs.append(
+                    Request(
+                        id=f"{row['day']}_{centre_hash}_{img_id}",
+                        raster_transform=rt,
+                        image=f"{df.attrs['collection']}/{img_id}",
+                        bands=df.attrs["bands"],
+                    )
+                )
+
+    return RequestSet(requestset=reqs)

{cubexpress-0.1.0.dist-info → cubexpress-0.1.1.dist-info}/METADATA

@@ -1,9 +1,10 @@
 Metadata-Version: 2.1
 Name: cubexpress
-Version: 0.1.0
-Summary: A Python package for efficient processing of cubic earth observation (EO) data
-Home-page: https://github.com/andesdatacube/cubexpress/
+Version: 0.1.1
+Summary: Efficient processing of cubic Earth-observation (EO) data.
+Home-page: https://github.com/andesdatacube/cubexpress
 License: MIT
+Keywords: earth-engine,sentinel-2,geospatial,eo,cube
 Author: Julio Contreras
 Author-email: contrerasnetk@gmail.com
 Requires-Python: >=3.9,<4.0
@@ -13,11 +14,18 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: GIS
+Provides-Extra: full
+Requires-Dist: earthengine-api (>=0.1.392) ; extra == "full"
 Requires-Dist: numpy (>=1.25.2)
 Requires-Dist: pandas (>=2.0.3)
+Requires-Dist: pyarrow (>=14.0.0) ; extra == "full"
+Requires-Dist: pygeohash (>=1.2.0,<2.0.0)
+Requires-Dist: rasterio (>=1.3.9) ; extra == "full"
 Requires-Dist: utm (>=0.8.0,<0.9.0)
-Project-URL: Documentation, https://andesdatacube.github.io/cubexpress/
-Project-URL: Repository, https://github.com/andesdatacube/cubexpress/
+Project-URL: Documentation, https://andesdatacube.github.io/cubexpress
+Project-URL: Repository, https://github.com/andesdatacube/cubexpress
 Description-Content-Type: text/markdown
 
 <h1></h1>

cubexpress-0.1.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+cubexpress/__init__.py,sha256=ybNczt27OVUkT8WE8v0-A9hByKtfNsNysg-V8CnZqxE,564
+cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
+cubexpress/cloud_utils.py,sha256=O4qcl8kr0_Yv0giZ-h4uzf791d1_z9HZy1Br8N867iA,8102
+cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
+cubexpress/cube.py,sha256=1GPVAt5Q0vGqA3QJ4gixAevlosV4JHNKMzN1PirhawI,2911
+cubexpress/downloader.py,sha256=u0u1LG2DOIaEvCPDIAaJDLH5_od52D1OPCbcpCicRzY,3320
+cubexpress/geospatial.py,sha256=ZbsPIgsYQFnNFXUuQ136rJsL4b2Bf91o0Vsswby2dFc,1812
+cubexpress/geotyping.py,sha256=6hjzjZhg6jRYRhLMQ_IiBygnShWlRCtpIbf6rRaQQ7s,17163
+cubexpress/request.py,sha256=cRm0J6Um8wCkbMDYBv9eCiqv32hLH28EH4eHLLDsJ-c,2333
+cubexpress-0.1.1.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
+cubexpress-0.1.1.dist-info/METADATA,sha256=3DhrMNjKWIfImjQgfCVRfade1JXcX2acDJX4iPtwR4U,9692
+cubexpress-0.1.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cubexpress-0.1.1.dist-info/RECORD,,

cubexpress/download.py

@@ -1,347 +0,0 @@
-import concurrent.futures
-import json
-import pathlib
-from concurrent.futures import ThreadPoolExecutor
-from copy import deepcopy
-from typing import Optional
-
-import ee
-import numpy as np
-import pandas as pd
-
-from cubexpress.geotyping import RequestSet
-
-
-def check_not_found_error(error_message: str) -> bool:
-    """
-    Checks if the error message indicates that the image was not found.
-
-    Args:
-        error_message (str): The error message to check.
-
-    Returns:
-        bool: True if the error message indicates "not found", False otherwise.
-
-    Example:
-        >>> check_not_found_error("Total request size must be less than or equal to...")
-        True
-    """
-    return (
-        "Total request size" in error_message
-        and "must be less than or equal to" in error_message
-    )
-
-
-def quadsplit_manifest(manifest: dict) -> list[dict]:
-    """
-    Splits a manifest into four smaller ones by dividing the grid dimensions.
-
-    Args:
-        manifest (dict): The original manifest to split.
-
-    Returns:
-        List[dict]: A list of four smaller manifests with updated grid transformations.
-
-    Example:
-        >>> manifest = {'grid': {'dimensions': {'width': 100, 'height': 100}, 'affineTransform': {'scaleX': 0.1, 'scaleY': 0.1, 'translateX': 0, 'translateY': 0}}}
-        >>> quadsplit_manifest(manifest)
-        [{'grid': {'dimensions': {'width': 50, 'height': 50}, 'affineTransform': {'scaleX': 0.1, 'scaleY': 0.1, 'translateX': 0, 'translateY': 0}}}, {'grid': {'dimensions': {'width': 50, 'height': 50}, 'affineTransform': {'scaleX': 0.1, 'scaleY': 0.1, 'translateX': 5.0, 'translateY': 0}}}, ...]
-    """
-    manifest_copy = deepcopy(manifest)
-    new_width = manifest["grid"]["dimensions"]["width"] // 2
-    new_height = manifest["grid"]["dimensions"]["height"] // 2
-    manifest_copy["grid"]["dimensions"]["width"] = new_width
-    manifest_copy["grid"]["dimensions"]["height"] = new_height
-
-    manifests = []
-    for idx in range(4):
-        new_manifest = deepcopy(manifest_copy)
-        res_x = manifest["grid"]["affineTransform"]["scaleX"]
-        res_y = manifest["grid"]["affineTransform"]["scaleY"]
-
-        add_x, add_y = (0, 0)
-        if idx == 1:
-            add_x = new_width * res_x
-        elif idx == 2:
-            add_y = new_height * res_y
-        elif idx == 3:
-            add_x = new_width * res_x
-            add_y = new_height * res_y
-
-        new_manifest["grid"]["affineTransform"]["translateX"] += add_x
-        new_manifest["grid"]["affineTransform"]["translateY"] += add_y
-
-        manifests.append(new_manifest)
-
-    return manifests
-
-
-def getGeoTIFFbatch(
-    manifest_dict: dict,
-    full_outname: pathlib.Path,
-    max_deep_level: Optional[int] = 5,
-    method: Optional[str] = "getPixels",
-) -> Optional[np.ndarray]:
-    """
-    Downloads a GeoTIFF image from Google Earth Engine using either the `getPixels` or `computePixels` method.
-    If the requested area exceeds the size limit, the image is recursively split into smaller tiles until the
-    download succeeds or the maximum recursion depth is reached.
-
-    Args:
-        manifest_dict (dict): A dictionary containing image metadata, including grid dimensions, affine transformations,
-                              and either an `assetId` or `expression` for the image source.
-        full_outname (pathlib.Path): The full path where the downloaded GeoTIFF file will be saved.
-        max_deep_level (Optional[int]): Maximum recursion depth for splitting large requests. Defaults to 5.
-        method (Optional[str]): Method for retrieving image data. Can be 'getPixels' for asset-based requests or
-                                'computePixels' for expressions. Defaults to 'getPixels'.
-
-    Returns:
-        Optional[pathlib.Path]: The path to the downloaded GeoTIFF file. Returns `None` if the download fails.
-
-    Raises:
-        ValueError: If the method is not 'getPixels' or 'computePixels', or if the image cannot be found.
-
-    Example:
-        >>> import ee
-        >>> import pathlib
-        >>> ee.Initialize()
-        >>> manifest_dict = {
-        ...     "assetId": "COPERNICUS/S2_HARMONIZED/20160816T153912_20160816T154443_T18TYN",
-        ...     "fileFormat": "GEO_TIFF",
-        ...     "bandIds": ["B4", "B3", "B2"],
-        ...     "grid": {
-        ...         "dimensions": {
-        ...             "width": 512,
-        ...             "height": 512
-        ...         },
-        ...         "affineTransform": {
-        ...             "scaleX": 10,
-        ...             "shearX": 0,
-        ...             "translateX": 725260.108545126,
-        ...             "scaleY": -10,
-        ...             "shearY": 0,
-        ...             "translateY": 4701550.38712196
-        ...         },
-        ...         "crsCode": "EPSG:32618"
-        ...     }
-        ... }
-
-        >>> getGeoTIFFbatch(manifest_dict pathlib.Path('output/sentinel_image.tif'))
-        PosixPath('output/sentinel_image.tif')
-    """
-
-    # Check if the maximum recursion depth has been reached
-    if max_deep_level == 0:
-        raise ValueError("Max recursion depth reached.")
-
-    try:
-        # Get the image bytes
-        if method == "getPixels":
-            image_bytes: bytes = ee.data.getPixels(manifest_dict)
-        elif method == "computePixels":
-            image_bytes: bytes = ee.data.computePixels(manifest_dict)
-        else:
-            raise ValueError("Method must be either 'getPixels' or 'computePixels'")
-
-        # Write the image bytes to a file
-        with open(full_outname, "wb") as src:
-            src.write(image_bytes)
-    except Exception as e:
-        # TODO: This is a workaround when the image is not found, as it is a message from the server
-        # it is not possible to check the type of the exception
-        if not check_not_found_error(str(e)):
-            raise ValueError(
-                f"Error downloading the GeoTIFF file from Earth Engine: {e}"
-            )
-
-        # Create the output directory if it doesn't exist
-        child_folder: pathlib.Path = full_outname.parent / full_outname.stem
-        pathlib.Path(child_folder).mkdir(parents=True, exist_ok=True)
-
-        # Split the manifest into four smaller manifests
-        manifest_dicts = quadsplit_manifest(manifest_dict)
-
-        for idx, manifest_dict_batch in enumerate(manifest_dicts):
-            # Recursively download the image
-            getGeoTIFFbatch(
-                full_outname=child_folder / ("%s__%02d.tif" % (full_outname.stem, idx)),
-                manifest_dict=manifest_dict_batch,
-                max_deep_level=max_deep_level - 1,
-                method=method,
-            )
-
-    return full_outname
-
-
-def getGeoTIFF(
-    manifest_dict: dict, full_outname: pathlib.Path, max_deep_level: Optional[int] = 5
-) -> Optional[np.ndarray]:
-    """
-    Retrieves an image from Earth Engine using the appropriate method based on the manifest type.
-
-    This function downloads a GeoTIFF image from Google Earth Engine (GEE). Depending on the content of
-    the provided manifest (`manifest_dict`), the function will either use the `getPixels` method (for
-    asset-based requests) or the `computePixels` method (for expressions). If the requested area exceeds
-    the size limit, the image will be recursively split into smaller tiles until the download succeeds or
-    the maximum recursion depth is reached.
-
-    Args:
-        manifest_dict (dict): A dictionary containing the image metadata. This should include either:
-            - `assetId`: The identifier of a GEE asset (e.g., satellite imagery).
-            - `expression`: A serialized string representing a GEE image expression (e.g., an image computation).
-            Additionally, the manifest should include grid information such as the image dimensions and affine transformations.
-
-        full_outname (pathlib.Path): The full path where the downloaded GeoTIFF file will be saved.
-
-        max_deep_level (Optional[int]): The maximum recursion depth for splitting large requests into smaller tiles if needed.
-            Defaults to 5.
-
-    Returns:
-        Optional[np.ndarray]: The downloaded image as a `numpy` array, or `None` if the download fails. It will
-            also return the full file path to the saved GeoTIFF image.
-
-    Raises:
-        ValueError: If the manifest does not contain either an `assetId` or `expression`, or if there is an error during download.
-
-    Example 1: Downloading an image using an `assetId`:
-        >>> import ee
-        >>> import pathlib
-        >>> ee.Initialize()
-        >>> manifest_dict = {
-        ...     "assetId": "COPERNICUS/S2_HARMONIZED/20160816T153912_20160816T154443_T18TYN",
-        ...     "fileFormat": "GEO_TIFF",
-        ...     "bandIds": ["B4", "B3", "B2"],
-        ...     "grid": {
-        ...         "dimensions": {"width": 512, "height": 512},
-        ...         "affineTransform": {
-        ...             "scaleX": 10,
-        ...             "shearX": 0,
-        ...             "translateX": 725260.108545126,
-        ...             "scaleY": -10,
-        ...             "shearY": 0,
-        ...             "translateY": 4701550.38712196
-        ...         },
-        ...         "crsCode": "EPSG:32618"
-        ...     }
-        ... }
-        >>> getGeoTIFF(manifest_dict, pathlib.Path('output/sentinel_image.tif'))
-        PosixPath('output/sentinel_image.tif')
-
-    Example 2: Downloading an image using an `expression`:
-        >>> image = ee.Image("COPERNICUS/S2_HARMONIZED/20160816T153912_20160816T154443_T18TYN") \
-        ...           .divide(10_000) \
-        ...           .select(["B4", "B3", "B2"])
-        >>> expression = image.serialize()
-        >>> manifest_dict = {
-        ...     "expression": expression,
-        ...     "fileFormat": "GEO_TIFF",
-        ...     "grid": {
-        ...         "dimensions": {"width": 512, "height": 512},
-        ...         "affineTransform": {
-        ...             "scaleX": 10,
-        ...             "shearX": 0,
-        ...             "translateX": 725260.108545126,
-        ...             "scaleY": -10,
-        ...             "shearY": 0,
-        ...             "translateY": 4701550.38712196
-        ...         },
-        ...         "crsCode": "EPSG:32618"
-        ...     }
-        ... }
-        >>> getGeoTIFF(manifest_dict, pathlib.Path('output/expression_image.tif'))
-        PosixPath('output/expression_image.tif')
-    """
-    if "assetId" in manifest_dict:
-        return getGeoTIFFbatch(
-            manifest_dict=manifest_dict,
-            full_outname=full_outname,
-            max_deep_level=max_deep_level,
-            method="getPixels",
-        )
-    elif "expression" in manifest_dict:
-        if isinstance(
-            manifest_dict["expression"], str
-        ):  # Decode only if the expression is still a string.
-            # From a string to a ee.Image object
-            manifest_dict["expression"] = ee.deserializer.decode(
-                json.loads(manifest_dict["expression"])
-            )
-
-        return getGeoTIFFbatch(
-            manifest_dict=manifest_dict,
-            full_outname=full_outname,
-            max_deep_level=max_deep_level,
-            method="computePixels",
-        )
-    else:
-        raise ValueError("Manifest does not contain 'assetId' or 'expression'")
-
-
-def getcube(
-    request: RequestSet,
-    output_path: str | pathlib.Path,
-    nworkers: Optional[int] = None,
-    max_deep_level: Optional[int] = 5,
-) -> list[pathlib.Path]:
-    """
-    Downloads multiple GeoTIFF images in parallel from Google Earth Engine (GEE) based on the provided request set.
-
-    Args:
-        request (RequestSet): A collection of image requests containing metadata and processing parameters.
-        output_path (Union[str, pathlib.Path]): Directory where the downloaded images will be saved.
-        nworkers (Optional[int], default=None): Number of parallel threads. If None, runs sequentially.
-        max_deep_level (Optional[int], default=5): Maximum recursion depth for image subdivision if exceeding GEE limits.
-
-    Returns:
-        List[pathlib.Path]: List of paths to the downloaded GeoTIFF files.
-
-    Example:
-        >>> import ee, cubexpress
-        >>> ee.Initialize()
-        >>> point = ee.Geometry.Point([-97.59, 33.37])
-        >>> collection = ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED") \
-        ...                 .filterBounds(point) \
-        ...                 .filterDate('2024-01-01', '2024-01-31')
-        >>> image_ids = collection.aggregate_array('system:id').getInfo()
-        >>> geotransform = cubexpress.lonlat2rt(lon=-97.59, lat=33.37, edge_size=128, scale=10)
-        >>> requests = [cubexpress.Request(id=f"s2_{i}", raster_transform=geotransform, bands=["B4", "B3", "B2"], image=ee.Image(img_id)) for i, img_id in enumerate(image_ids)]
-        >>> cube_requests = cubexpress.RequestSet(requestset=requests)
-        >>> cubexpress.getcube(request=cube_requests, nworkers=4, output_path="output", max_deep_level=5)
-        [PosixPath('output/s2_0.tif'), PosixPath('output/s2_1.tif'), ...]
-    """
-
-    # Check that _dataframe exists and is not empty
-    if request._dataframe is None or request._dataframe.empty:
-        raise ValueError(
-            "The request's _dataframe is None or empty. "
-            "There are no valid requests to process."
-        )
-    
-    # **Revalidate** the DataFrame structure, in case the user manipulated it.
-    request._validate_dataframe_schema()
-
-    # Get the table
-    table: pd.DataFrame = request._dataframe
-
-    # Create the output directory if it doesn't exist
-    output_path = pathlib.Path(output_path)
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    results = []
-    with ThreadPoolExecutor(max_workers=nworkers) as executor:
-        futures = {
-            executor.submit(
-                getGeoTIFF, row.manifest, output_path / row.outname, max_deep_level
-            ): row
-            for _, row in table.iterrows()
-        }
-        for future in concurrent.futures.as_completed(futures):
-            try:
-                result = future.result()
-                if result:
-                    results.append(result)
-            except Exception as e:
-                # TODO add this into the log
-                print(f"Error processing {futures[future].outname}: {e}")
-
-    return results

cubexpress-0.1.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-cubexpress/__init__.py,sha256=1CF6kINn70mfS5HNzYyTf4UsOUPG0qzeetoJSDk0ALw,418
-cubexpress/conversion.py,sha256=h77re8AtdVV_Jy3ugZeQ-e2I8DHSKoghiq70MXkzBaQ,2506
-cubexpress/download.py,sha256=DX5DKPdKiuv1gHxs-5Q5ScZ06nvE-Pi1YGLSzQc2jrs,14315
-cubexpress/geotyping.py,sha256=5JgsOfRfwQf-iBh902wKQ1AxEKw1HgFL2brzwkxO0Pg,17152
-cubexpress-0.1.0.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
-cubexpress-0.1.0.dist-info/METADATA,sha256=XfBIfpFP1quHSNr60Dn6R8EEpdq02XJWCepwhl7j7U0,9327
-cubexpress-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cubexpress-0.1.0.dist-info/RECORD,,