cubexpress 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

cubexpress/__init__.py

@@ -1,6 +1,6 @@
 from cubexpress.conversion import lonlat2rt, geo2utm
 from cubexpress.geotyping import RasterTransform, Request, RequestSet
-from cubexpress.cloud_utils import cloud_table
+from cubexpress.cloud_utils import s2_cloud_table
 from cubexpress.cube import get_cube
 from cubexpress.request import table_to_requestset
 
@@ -15,7 +15,7 @@ __all__ = [
     "RequestSet",
     "geo2utm",
     "get_cube",
-    "cloud_table",
+    "s2_cloud_table",
     "table_to_requestset"
 ]
 

cubexpress/cloud_utils.py

@@ -27,25 +27,30 @@ def _cloud_table_single_range(
     start: str,
     end: str
 ) -> pd.DataFrame:
-    """Return raw cloud-table rows for a single *start–end* interval.
+    """
+    Build a daily cloud-score table for a square Sentinel-2 footprint.
 
     Parameters
     ----------
-    lon, lat
-        Centre coordinates in decimal degrees.
-    edge_size, scale
-        ROI size in pixels (*edge_size*) and pixel resolution in metres
-        (*scale*), fed into :pyfunc:`cubexpress.geospatial._square_roi`.
-    start, end
-        ISO-dates (``YYYY-MM-DD``) delimiting the query.
-    collection
-        Sentinel-2 collection name to query.
+    lon, lat : float
+        Point at the centre of the requested region (°).
+    edge_size : int
+        Side length of the square region in Sentinel-2 pixels (10 m each).
+    start, end : str
+        ISO-8601 dates delimiting the period, e.g. ``"2024-06-01"``.
 
     Returns
     -------
     pandas.DataFrame
-        Columns: **day** (str), **cloudPct** (float), **images** (str
-        concatenation of asset IDs separated by ``-``). No filtering applied.
+        One row per image with columns:
+        * ``id`` – Sentinel-2 ID
+        * ``cs_cdf`` – Cloud Score Plus CDF (0–1)
+        * ``date`` – acquisition date (YYYY-MM-DD)
+        * ``null_flag`` – 1 if cloud score missing
+
+    Notes
+    -----
+    Missing ``cs_cdf`` values are filled with the mean of the same day.
     """
 
     center = ee.Geometry.Point([lon, lat])
@@ -64,6 +69,8 @@ def _cloud_table_single_range(
         .linkCollection(csp, ["cs_cdf"])
         .select(["cs_cdf"])
     )
+
+    # image IDs for every expected date
     ids = ic.aggregate_array("system:index").getInfo()
     df_ids = pd.DataFrame({"id": ids})
 
@@ -76,7 +83,7 @@ def _cloud_table_single_range(
     except ee.ee_exception.EEException as e:
         if "No bands in collection" in str(e):
             return pd.DataFrame(
-                columns=["id", "cs_cdf", "date", "high_null_flag"]
+                columns=["id", "cs_cdf", "date", "null_flag"]
             )
         raise 
 
@@ -88,11 +95,12 @@ def _cloud_table_single_range(
         .merge(df_raw, on="id", how="left")
         .assign(
             date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d"),
-            high_null_flag=lambda d: d["cs_cdf"].isna().astype(int),
+            null_flag=lambda d: d["cs_cdf"].isna().astype(int),
         )
         .drop(columns=["longitude", "latitude", "time"])
     )
 
+    # fill missing scores with daily mean
     df["cs_cdf"] = df["cs_cdf"].fillna(df.groupby("date")["cs_cdf"].transform("mean"))
 
     return df
@@ -101,11 +109,12 @@ def _cloud_table_single_range(
 def s2_cloud_table(
     lon: float,
     lat: float,
-    edge_size: int = 2048,
-    start: str = "2017-01-01",
-    end: str = "2024-12-31",
-    cscore: float = 0.5,
-    cache: bool = True,
+    edge_size: int,
+    start: str,
+    end: str,
+    max_cscore: float = 1.0,
+    min_cscore: float = 0.0,
+    cache: bool = False,
     verbose: bool = True,
 ) -> pd.DataFrame:
     """Build (and cache) a per-day cloud-table for the requested ROI.
@@ -152,7 +161,7 @@ def s2_cloud_table(
     # ─── 1. Load cached data if present ────────────────────────────────────
     if cache and cache_file.exists():
         if verbose:
-            print("📂  Loading cached table …")
+            print("📂  Loading cached metadata …")
         df_cached = pd.read_parquet(cache_file)
         have_idx = pd.to_datetime(df_cached["date"], errors="coerce").dropna()
 
@@ -164,7 +173,7 @@ def s2_cloud_table(
             and dt.date.fromisoformat(end) <= cached_end
         ):
             if verbose:
-                print("✅  Served entirely from cache.")
+                print("✅  Served entirely from metadata.")
             df_full = df_cached
         else:
             # Identify missing segments and fetch only those.
@@ -183,15 +192,21 @@ def s2_cloud_table(
                         lon, lat, edge_size, a2, b2
                     )
                 )
-            df_new = pd.concat(df_new_parts, ignore_index=True)
-            df_full = (
-                pd.concat([df_cached, df_new], ignore_index=True)
-                .sort_values("date", kind="mergesort")
-            )
+            df_new_parts = [df for df in df_new_parts if not df.empty]
+            
+            if df_new_parts:
+                
+                df_new = pd.concat(df_new_parts, ignore_index=True)
+                df_full = (
+                    pd.concat([df_cached, df_new], ignore_index=True)
+                    .sort_values("date", kind="mergesort")
+                )
+            else:
+                df_full = df_cached
     else:
 
         if verbose:
-            msg = "Generating table (no cache found)…" if cache else "Generating table…"
+            msg = "Generating metadata (no cache found)…" if cache else "Generating metadata…"
             print("⏳", msg)
         df_full = _cloud_table_single_range(
             lon, lat, edge_size, start, end
@@ -206,7 +221,7 @@ def s2_cloud_table(
     
     result = (
         df_full.query("@start <= date <= @end")
-        .query("cs_cdf > @cscore")
+        .query("@min_cscore <= cs_cdf <= @max_cscore")
         .reset_index(drop=True)
     )
 
@@ -222,3 +237,4 @@ def s2_cloud_table(
         }
     )
     return result
+

cubexpress/cube.py

@@ -16,17 +16,21 @@ from __future__ import annotations
 import pathlib
 import concurrent.futures
 from typing import Dict, Any
-
 import ee
+
+
 from cubexpress.downloader import download_manifest, download_manifests
 from cubexpress.geospatial import quadsplit_manifest, calculate_cell_size
-from cubexpress.geotyping import RequestSet
+from cubexpress.request import table_to_requestset
+import pandas as pd
 
 
 def get_geotiff(
     manifest: Dict[str, Any],
     full_outname: pathlib.Path | str,
+    join: bool = True,
     nworks: int = 4,
+    verbose: bool = True,
 ) -> None:
     """Download *manifest* to *full_outname*, retrying with tiled requests.
 
@@ -39,20 +43,28 @@ def get_geotiff(
     nworks
         Maximum worker threads when the image must be split; default **4**.
     """
+    full_outname = pathlib.Path(full_outname)
     try:
-        download_manifest(manifest, pathlib.Path(full_outname))
+        download_manifest(manifest, full_outname)
     except ee.ee_exception.EEException as err:
-        # Handle EE “too many pixels” error by recursive tiling.
+
         size = manifest["grid"]["dimensions"]["width"]  # square images assumed
         cell_w, cell_h, power = calculate_cell_size(str(err), size)
         tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
-        download_manifests(tiled, max_workers=nworks, full_outname=pathlib.Path(full_outname))
+        download_manifests(tiled, full_outname, join, nworks)
+
+    if verbose:
+        print(f"Downloaded {full_outname}")
 
 
 def get_cube(
-    requests: RequestSet,
+    table: pd.DataFrame,
     outfolder: pathlib.Path | str,
+    mosaic: bool = True,
+    join: bool = True,
     nworks: int = 4,
+    verbose: bool = True,
+    cache: bool = True
 ) -> None:
     """Download every request in *requests* to *outfolder* using a thread pool.
 
@@ -68,17 +80,40 @@ def get_cube(
     nworks
         Pool size for concurrent downloads; default **4**.
     """
-    out = pathlib.Path(outfolder)
+
+    requests = table_to_requestset(
+        table=table, 
+        mosaic=mosaic
+    )
+    
+    outfolder = pathlib.Path(outfolder).expanduser().resolve()
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=nworks) as pool:
         futures = []
         for _, row in requests._dataframe.iterrows():
-            outname = out / f"{row.id}.tif"
+            outname = pathlib.Path(outfolder) / f"{row.id}.tif"
+            if outname.exists() and cache:
+                continue
             outname.parent.mkdir(parents=True, exist_ok=True)
-            futures.append(pool.submit(get_geotiff, row.manifest, outname, nworks))
+            futures.append(
+                pool.submit(
+                    get_geotiff, 
+                    row.manifest, 
+                    outname, 
+                    join, 
+                    nworks, 
+                    verbose
+                )
+            )
 
         for fut in concurrent.futures.as_completed(futures):
             try:
                 fut.result()
             except Exception as exc:  # noqa: BLE001 – log and keep going
                 print(f"Download error: {exc}")
+
+    download_df = requests._dataframe[["outname", "cs_cdf", "date"]].copy()
+    download_df["outname"] = outfolder / requests._dataframe["outname"]
+    download_df.rename(columns={"outname": "full_outname"}, inplace=True)
+
+    return download_df

cubexpress/downloader.py

@@ -21,7 +21,11 @@ import ee
 import rasterio as rio
 from rasterio.io import MemoryFile
 import logging
+from rasterio.merge import merge
+from rasterio.enums import Resampling
 import os
+import shutil
+import tempfile
 
 os.environ['CPL_LOG_ERRORS'] = 'OFF'
 logging.getLogger('rasterio._env').setLevel(logging.ERROR)
@@ -50,10 +54,10 @@ def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None
                 driver="GTiff", 
                 tiled=True,
                 interleave="band",
-                blockxsize=256,
+                blockxsize=256, # TODO: Creo que es 128 (por de la superresolucion)
                 blockysize=256,
                 compress="ZSTD",
-                zstd_level=13,
+                # zstd_level=13,
                 predictor=2,
                 num_threads=20,
                 nodata=65535,
@@ -65,13 +69,11 @@ def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None
             with rio.open(full_outname, "w", **profile) as dst:
                 dst.write(src.read())
 
-    print(f"{full_outname} downloaded successfully.")  # noqa: T201
-
-
 def download_manifests(
-    manifests: List[Dict[str, Any]],
-    max_workers: int,
+    manifests: list[Dict[str, Any]],
     full_outname: pathlib.Path,
+    join: bool = True,
+    max_workers: int = 4,
 ) -> None:
     """Download every manifest in *manifests* concurrently.
 
@@ -79,6 +81,12 @@ def download_manifests(
     ``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
     ``000001.tif`` … according to the list order.
     """
+    # full_outname = pathlib.Path("/home/contreras/Documents/GitHub/cubexpress/cubexpress_test/2017-08-19_6mfrw_18LVN.tif")
+    original_dir = full_outname.parent
+    if join:
+        tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="s2tmp_"))
+        full_outname = tmp_dir / full_outname.name
+
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         futures = []
 
@@ -93,3 +101,35 @@ def download_manifests(
                 fut.result()
             except Exception as exc:  # noqa: BLE001
                 print(f"Error en una de las descargas: {exc}")  # noqa: T201
+
+    dir_path = full_outname.parent / full_outname.stem
+    input_files = sorted(dir_path.glob("*.tif"))
+
+    if dir_path.exists() and len(input_files) > 1:
+
+        with rio.Env(GDAL_NUM_THREADS="8", NUM_THREADS="8"):
+            srcs = [rio.open(fp) for fp in input_files]
+            mosaic, out_transform = merge(
+                srcs,
+                nodata=65535,
+                resampling=Resampling.nearest
+            )
+
+            meta = srcs[0].profile.copy()
+            meta["transform"] = out_transform
+            meta.update(
+                height=mosaic.shape[1],
+                width=mosaic.shape[2]
+            )
+            outname = original_dir / full_outname.name
+            outname.parent.mkdir(parents=True, exist_ok=True)
+            with rio.open(outname, "w", **meta) as dst:
+                dst.write(mosaic)
+
+            for src in srcs:
+                src.close()
+
+        # Delete a folder with pathlib
+        shutil.rmtree(dir_path) 
+    else:
+        return outname

cubexpress/geotyping.py

@@ -306,13 +306,17 @@ class RequestSet(BaseModel):
                             "crsCode": meta.raster_transform.crs,
                         },
                     },
+                    "cs_cdf": int(meta.id.split("_")[-1]) / 100,
+                    "date": meta.id.split("_")[0],
                     "outname": f"{meta.id}.tif",
                 }
+                
                 for index, meta in enumerate(self.requestset)
             ]
         )
 
 
+
     def _validate_dataframe_schema(self) -> None:
         """
         Checks that the `_dataframe` contains the required columns and that each column
@@ -367,21 +371,7 @@ class RequestSet(BaseModel):
                             f"Column '{col_name}' has an invalid type in row {i}. "
                             f"Expected {expected_type}, got {type(value)}"
                         )
-
-        # B) Validation of the `manifest` column structure
-        #    - Must contain at least 'assetId' or 'expression'
-        #    - Must contain 'grid' with the minimum required sub-keys
-        #    - Example:
-        #         {
-        #           "fileFormat": "GEO_TIFF",
-        #           "bandIds": [...],
-        #           "grid": {
-        #              "dimensions": {"width": ..., "height": ...},
-        #              "affineTransform": {...},
-        #              "crsCode": ...
-        #           },
-        #           // Either "assetId" or "expression" must be here
-        #         }
+                    
         for i, row in self._dataframe.iterrows():
             manifest = row["manifest"]
 
@@ -482,7 +472,7 @@ class RequestSet(BaseModel):
             str: A string representation of the entire RasterTransformSet.
         """
         num_entries = len(self.requestset)
-        return f"RasterTransformSet({num_entries} entries)"
+        return f"RequestSet({num_entries} entries)"
 
     def __str__(self):
         return super().__repr__()

cubexpress/request.py

@@ -5,13 +5,15 @@ from __future__ import annotations
 import ee
 import pandas as pd
 import pygeohash as pgh
-from typing import List
 
 from cubexpress.geotyping import Request, RequestSet
 from cubexpress.conversion import lonlat2rt
 
 
-def table_to_requestset(df: pd.DataFrame, *, mosaic: bool = True) -> RequestSet:
+def table_to_requestset(
+        table: pd.DataFrame, 
+        mosaic: bool = True
+    ) -> RequestSet:
     """Return a :class:`RequestSet` built from *df* (cloud_table result).
 
     Parameters
@@ -30,52 +32,75 @@ def table_to_requestset(df: pd.DataFrame, *, mosaic: bool = True) -> RequestSet:
 
     """
 
+    
+    df = table.copy()
 
-    df_ = df.copy()
-
-    if df_.empty:
+    if df.empty:
         raise ValueError("cloud_table returned no rows; nothing to request.")
 
     rt = lonlat2rt(
-        lon=df_.attrs["lon"],
-        lat=df_.attrs["lat"],
-        edge_size=df_.attrs["edge_size"],
-        scale=df_.attrs["scale"],
+        lon=df.attrs["lon"],
+        lat=df.attrs["lat"],
+        edge_size=df.attrs["edge_size"],
+        scale=df.attrs["scale"],
     )
-    centre_hash = pgh.encode(df_.attrs["lat"], df_.attrs["lon"], precision=5)
+    centre_hash = pgh.encode(df.attrs["lat"], df.attrs["lon"], precision=5)
     reqs: list[Request] = []
 
+
+
     if mosaic:
-        # group all asset IDs per day
         grouped = (
-            df_.groupby("date")["id"]   # Series con listas de ids por día
-            .apply(list)
+        df.groupby('date')
+            .agg(
+                id_list      = ('id', list),
+                cs_cdf_mean  = ('cs_cdf', lambda x: int(round(x.mean(), 2) * 100))
+            )
         )
 
-        for day, img_ids in grouped.items():
-            ee_img = ee.ImageCollection(
-                [ee.Image(f"{df_.attrs['collection']}/{img}") for img in img_ids]
-            ).mosaic()
-
-            reqs.append(
-                Request(
-                    id=f"{day}_{centre_hash}",
-                    raster_transform=rt,
-                    image=ee_img,
-                    bands=df_.attrs["bands"],
+        for day, row in grouped.iterrows():
+            
+            img_ids   = row["id_list"]
+            cdf  = row["cs_cdf_mean"]
+            
+            if len(img_ids) > 1:
+
+                ee_img = ee.ImageCollection(
+                    [ee.Image(f"{df.attrs['collection']}/{img}") for img in img_ids]
+                ).mosaic()
+
+                reqs.append(
+                    Request(
+                        id=f"{day}_{centre_hash}_{cdf}",
+                        raster_transform=rt,
+                        image=ee_img,
+                        bands=df.attrs["bands"],
+                    )
                 )
-            )
-    else:  # one request per asset
-        for _, row in df_.iterrows():
+            else:
+                for img_id in img_ids:
+                    tile = img_id.split("_")[-1][1:]
+                    reqs.append(
+                        Request(
+                            id=f"{day}_{centre_hash}_{tile}_{cdf}",
+                            raster_transform=rt,
+                            image=f"{df.attrs['collection']}/{img_id}",
+                            bands=df.attrs["bands"],
+                        )
+                    )
+    else:
+        for _, row in df.iterrows():
             img_id = row["id"]
-            day    = row["date"]
+            tile = img_id.split("_")[-1][1:]
+            day = row["date"]
+            cdf = int(round(row["cs_cdf"], 2) * 100)
 
             reqs.append(
                 Request(
-                    id=f"{day}_{centre_hash}_{img_id}",
+                    id=f"{day}_{centre_hash}_{tile}_{cdf}",
                     raster_transform=rt,
-                    image=f"{df_.attrs['collection']}/{img_id}",
-                    bands=df_.attrs["bands"],
+                    image=f"{df.attrs['collection']}/{img_id}",
+                    bands=df.attrs["bands"],
                 )
             )
 

{cubexpress-0.1.8.dist-info → cubexpress-0.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cubexpress
-Version: 0.1.8
+Version: 0.1.10
 Summary: Efficient processing of cubic Earth-observation (EO) data.
 Home-page: https://github.com/andesdatacube/cubexpress
 License: MIT
@@ -20,6 +20,7 @@ Requires-Dist: earthengine-api (>=1.5.12)
 Requires-Dist: numpy (>=2.0.2)
 Requires-Dist: pandas (>=2.2.2)
 Requires-Dist: pyarrow (>=14.0.0)
+Requires-Dist: pydantic (>=2.11.4)
 Requires-Dist: pygeohash (>=1.2.0)
 Requires-Dist: pyproj (>=3.6.0)
 Requires-Dist: rasterio (>=1.3.9)

cubexpress-0.1.10.dist-info/RECORD

@@ -0,0 +1,13 @@
+cubexpress/__init__.py,sha256=RjyAqwiD0rU_Z5tCJTYNGKXZ1ggpfPB51wzhr0KwweY,570
+cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
+cubexpress/cloud_utils.py,sha256=Vr2A1SZDKP_2xNiLYgwmWOUX8P8I-pXQrxBETiUDq60,7441
+cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
+cubexpress/cube.py,sha256=tU0lqhtQUwEiz33yebYIbw-a0R4zmTAei-b_xqMIcWU,3719
+cubexpress/downloader.py,sha256=gHVNCNTwK9qA5MPaEHB_m0wOPprw010qaTVnszwbuUk,4668
+cubexpress/geospatial.py,sha256=ZbsPIgsYQFnNFXUuQ136rJsL4b2Bf91o0Vsswby2dFc,1812
+cubexpress/geotyping.py,sha256=XoSXQuoq5CfzKndM2Pko5KXIP0vxGNm02LOOMbCWkrs,16692
+cubexpress/request.py,sha256=jy5K9MQEurNlwhF0izFmoIh3o7m9bC97fsTT_7C7Gv0,3051
+cubexpress-0.1.10.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
+cubexpress-0.1.10.dist-info/METADATA,sha256=Tn_XBaLWbO4xbmzYJCM6vnbwRNZ1d1ABZ6uF4G4REYM,9664
+cubexpress-0.1.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cubexpress-0.1.10.dist-info/RECORD,,

cubexpress-0.1.8.dist-info/RECORD

@@ -1,13 +0,0 @@
-cubexpress/__init__.py,sha256=ybNczt27OVUkT8WE8v0-A9hByKtfNsNysg-V8CnZqxE,564
-cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
-cubexpress/cloud_utils.py,sha256=svK4XpXj2CcyGqGwZryxmlRxrl8Z45qtvcAf2VMAaxk,7041
-cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
-cubexpress/cube.py,sha256=1GPVAt5Q0vGqA3QJ4gixAevlosV4JHNKMzN1PirhawI,2911
-cubexpress/downloader.py,sha256=JWRegfrxWaFSrYOkobV7IeAZNi6aS6VXvm3mwj5QJSw,3255
-cubexpress/geospatial.py,sha256=ZbsPIgsYQFnNFXUuQ136rJsL4b2Bf91o0Vsswby2dFc,1812
-cubexpress/geotyping.py,sha256=6hjzjZhg6jRYRhLMQ_IiBygnShWlRCtpIbf6rRaQQ7s,17163
-cubexpress/request.py,sha256=9Xhm98zQrJx73PCcy-dzbFAjVQ08O0lHDH4hXLOLuwM,2282
-cubexpress-0.1.8.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
-cubexpress-0.1.8.dist-info/METADATA,sha256=uNYE-wmqWuaGEJpOBwdFa3scYZNu9GwB4My6oumg_6U,9628
-cubexpress-0.1.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cubexpress-0.1.8.dist-info/RECORD,,