cubexpress 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

cubexpress/__init__.py

@@ -1,5 +1,5 @@
 from cubexpress.conversion import lonlat2rt, geo2utm
-from cubexpress.geotyping import RasterTransform, Request, RequestSet
+from cubexpress.geotyping import RasterTransform, Request, RequestSet, GeotransformDict
 from cubexpress.cloud_utils import s2_cloud_table
 from cubexpress.cube import get_cube
 from cubexpress.request import table_to_requestset
@@ -11,6 +11,7 @@ from cubexpress.request import table_to_requestset
 __all__ = [
     "lonlat2rt",
     "RasterTransform",
+    "GeotransformDict",
     "Request",
     "RequestSet",
     "geo2utm",
@@ -19,7 +20,7 @@ __all__ = [
     "table_to_requestset"
 ]
 
-# Dynamic version import
-import importlib.metadata
+# # Dynamic version import
+# import importlib.metadata
 
-__version__ = importlib.metadata.version("cubexpress")
+# __version__ = importlib.metadata.version("cubexpress")

cubexpress/cloud_utils.py

@@ -46,7 +46,7 @@ def _cloud_table_single_range(
         * ``id`` – Sentinel-2 ID
         * ``cs_cdf`` – Cloud Score Plus CDF (0–1)
         * ``date`` – acquisition date (YYYY-MM-DD)
-        * ``high_null_flag`` – 1 if cloud score missing
+        * ``null_flag`` – 1 if cloud score missing
 
     Notes
     -----
@@ -83,7 +83,7 @@ def _cloud_table_single_range(
     except ee.ee_exception.EEException as e:
         if "No bands in collection" in str(e):
             return pd.DataFrame(
-                columns=["id", "cs_cdf", "date", "high_null_flag"]
+                columns=["id", "cs_cdf", "date", "null_flag"]
             )
         raise 
 
@@ -95,14 +95,17 @@ def _cloud_table_single_range(
         .merge(df_raw, on="id", how="left")
         .assign(
             date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d"),
-            high_null_flag=lambda d: d["cs_cdf"].isna().astype(int),
+            null_flag=lambda d: d["cs_cdf"].isna().astype(int),
         )
         .drop(columns=["longitude", "latitude", "time"])
     )
 
     # fill missing scores with daily mean
+    df["lon"] = lon
+    df["lat"] = lat
     df["cs_cdf"] = df["cs_cdf"].fillna(df.groupby("date")["cs_cdf"].transform("mean"))
 
+
     return df
 
 
@@ -161,7 +164,7 @@ def s2_cloud_table(
     # ─── 1. Load cached data if present ────────────────────────────────────
     if cache and cache_file.exists():
         if verbose:
-            print("📂  Loading cached table …")
+            print("📂  Loading cached metadata …")
         df_cached = pd.read_parquet(cache_file)
         have_idx = pd.to_datetime(df_cached["date"], errors="coerce").dropna()
 
@@ -173,7 +176,7 @@ def s2_cloud_table(
             and dt.date.fromisoformat(end) <= cached_end
         ):
             if verbose:
-                print("✅  Served entirely from cache.")
+                print("✅  Served entirely from metadata.")
             df_full = df_cached
         else:
             # Identify missing segments and fetch only those.
@@ -192,15 +195,21 @@ def s2_cloud_table(
                         lon, lat, edge_size, a2, b2
                     )
                 )
-            df_new = pd.concat(df_new_parts, ignore_index=True)
-            df_full = (
-                pd.concat([df_cached, df_new], ignore_index=True)
-                .sort_values("date", kind="mergesort")
-            )
+            df_new_parts = [df for df in df_new_parts if not df.empty]
+            
+            if df_new_parts:
+                
+                df_new = pd.concat(df_new_parts, ignore_index=True)
+                df_full = (
+                    pd.concat([df_cached, df_new], ignore_index=True)
+                    .sort_values("date", kind="mergesort")
+                )
+            else:
+                df_full = df_cached
     else:
 
         if verbose:
-            msg = "Generating table (no cache found)…" if cache else "Generating table…"
+            msg = "Generating metadata (no cache found)…" if cache else "Generating metadata…"
             print("⏳", msg)
         df_full = _cloud_table_single_range(
             lon, lat, edge_size, start, end
@@ -230,4 +239,5 @@ def s2_cloud_table(
             "collection": collection
         }
     )
-    return result
+    return result
+

cubexpress/cube.py

@@ -23,13 +23,13 @@ from cubexpress.downloader import download_manifest, download_manifests
 from cubexpress.geospatial import quadsplit_manifest, calculate_cell_size
 from cubexpress.request import table_to_requestset
 import pandas as pd
+from cubexpress.geotyping import RequestSet
 
 
 def get_geotiff(
     manifest: Dict[str, Any],
     full_outname: pathlib.Path | str,
     join: bool = True,
-    eraser: bool = True,
     nworks: int = 4,
     verbose: bool = True,
 ) -> None:
@@ -52,19 +52,26 @@ def get_geotiff(
         size = manifest["grid"]["dimensions"]["width"]  # square images assumed
         cell_w, cell_h, power = calculate_cell_size(str(err), size)
         tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
-        download_manifests(tiled, full_outname, join, eraser, nworks)
+        download_manifests(
+            manifests = tiled, 
+            full_outname = full_outname, 
+            join = join, 
+            max_workers = nworks
+        )
 
     if verbose:
         print(f"Downloaded {full_outname}")
 
 
 def get_cube(
-    table: pd.DataFrame,
+    # table: pd.DataFrame,
+    requests: pd.DataFrame | RequestSet,
     outfolder: pathlib.Path | str,
-    join: bool = True,
-    eraser: bool = True,
     mosaic: bool = True,
+    join: bool = True,
     nworks: int = 4,
+    verbose: bool = True,
+    cache: bool = True
 ) -> None:
     """Download every request in *requests* to *outfolder* using a thread pool.
 
@@ -81,20 +88,45 @@ def get_cube(
         Pool size for concurrent downloads; default **4**.
     """
 
-    requests = table_to_requestset(
-        table=table, 
-        mosaic=mosaic
-    )
+    # requests = table_to_requestset(
+    #     table=table, 
+    #     mosaic=mosaic
+    # )
+    
+    outfolder = pathlib.Path(outfolder).expanduser().resolve()
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=nworks) as pool:
         futures = []
         for _, row in requests._dataframe.iterrows():
             outname = pathlib.Path(outfolder) / f"{row.id}.tif"
+            if outname.exists() and cache:
+                continue
             outname.parent.mkdir(parents=True, exist_ok=True)
-            futures.append(pool.submit(get_geotiff, row.manifest, outname, join, eraser, nworks))
+            futures.append(
+                pool.submit(
+                    get_geotiff, 
+                    row.manifest, # manifest = row.manifest
+                    outname, # full_outname = outname
+                    join, # join = join
+                    nworks, # nworks = nworks
+                    verbose # verbose = verbose
+                )
+            )
 
         for fut in concurrent.futures.as_completed(futures):
             try:
                 fut.result()
             except Exception as exc:  # noqa: BLE001 – log and keep going
                 print(f"Download error: {exc}")
+
+    # download_df = requests._dataframe[["outname", "cs_cdf", "date"]].copy()
+    # download_df["outname"] = outfolder / requests._dataframe["outname"]
+    # download_df.rename(columns={"outname": "full_outname"}, inplace=True)
+
+    return 
+
+# manifest = row.manifest
+# full_outname = outname
+# join: bool = True,
+# nworks: int = 4,
+# verbose: bool = True,

cubexpress/downloader.py

@@ -25,6 +25,8 @@ from rasterio.merge import merge
 from rasterio.enums import Resampling
 import os
 import shutil
+import tempfile
+from cubexpress.geospatial import merge_tifs
 
 os.environ['CPL_LOG_ERRORS'] = 'OFF'
 logging.getLogger('rasterio._env').setLevel(logging.ERROR)
@@ -56,12 +58,12 @@ def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None
                 blockxsize=256,
                 blockysize=256,
                 compress="ZSTD",
-                # zstd_level=13,
+                zstd_level=13,
                 predictor=2,
                 num_threads=20,
                 nodata=65535,
                 dtype="uint16",
-                count=13,
+                count=12,
                 photometric="MINISBLACK"
             )
 
@@ -69,10 +71,9 @@ def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None
                 dst.write(src.read())
 
 def download_manifests(
-    manifests: List[Dict[str, Any]],
+    manifests: list[Dict[str, Any]],
     full_outname: pathlib.Path,
     join: bool = True,
-    eraser: bool = True,
     max_workers: int = 4,
 ) -> None:
     """Download every manifest in *manifests* concurrently.
@@ -81,55 +82,38 @@ def download_manifests(
     ``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
     ``000001.tif`` … according to the list order.
     """
+    # full_outname = pathlib.Path("/home/contreras/Documents/GitHub/cubexpress/cubexpress_test/2017-08-19_6mfrw_18LVN.tif")
+
+    if join:
+        tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="s2tmp_"))
+        full_outname_temp = tmp_dir / full_outname.name
+
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         futures = []
 
         for index, umanifest in enumerate(manifests):
-            folder = full_outname.parent / full_outname.stem
+            folder = full_outname_temp.parent / full_outname_temp.stem
             folder.mkdir(parents=True, exist_ok=True)
             outname = folder / f"{index:06d}.tif"
-            futures.append(executor.submit(download_manifest, umanifest, outname))
+            futures.append(
+                executor.submit(
+                    download_manifest, 
+                    umanifest, # ulist = umanifest
+                    outname # full_outname = outname
+                )
+         )
 
         for fut in concurrent.futures.as_completed(futures):
             try:
                 fut.result()
             except Exception as exc:  # noqa: BLE001
                 print(f"Error en una de las descargas: {exc}")  # noqa: T201
-    
-    if join:
-
-        dir_path = full_outname.parent / full_outname.stem
-        input_files = sorted(dir_path.glob("*.tif"))
-
-        if dir_path.exists() and len(input_files) > 1:
 
-            with rio.Env(GDAL_NUM_THREADS="8", NUM_THREADS="8"):
-                srcs = [rio.open(fp) for fp in input_files]
-                mosaic, out_transform = merge(
-                    srcs,
-                    nodata=65535,
-                    resampling=Resampling.nearest
-                )
-
-                meta = srcs[0].profile.copy()
-                meta["transform"] = out_transform
-                meta.update(
-                    height=mosaic.shape[1],
-                    width=mosaic.shape[2]
-                )
-
-                with rio.open(full_outname, "w", **meta) as dst:
-                    dst.write(mosaic)
 
-                for src in srcs:
-                    src.close()
-
-            if eraser:
-                # Delete a folder with pathlib
-                shutil.rmtree(dir_path) 
-
-            print("✅ Mosaico generado:", full_outname)
-            return full_outname
-
-        else:
-            return full_outname
+    dir_path = full_outname_temp.parent / full_outname_temp.stem
+    if dir_path.exists():
+        input_files = sorted(dir_path.glob("*.tif"))
+        merge_tifs(input_files, full_outname)
+        shutil.rmtree(dir_path) 
+    else:
+        raise ValueError(f"Error in {full_outname}")

cubexpress/geospatial.py

@@ -2,6 +2,11 @@ import ee
 import re
 from copy import deepcopy
 from typing import Dict
+import pathlib
+import rasterio as rio
+from rasterio.merge import merge
+from rasterio.enums import Resampling
+
 
 
 def quadsplit_manifest(manifest: Dict, cell_width: int, cell_height: int, power: int) -> list[Dict]:
@@ -27,8 +32,6 @@ def quadsplit_manifest(manifest: Dict, cell_width: int, cell_height: int, power:
 
     return manifests
 
-
-
 def calculate_cell_size(ee_error_message: str, size: int) -> tuple[int, int]:
     match = re.findall(r'\d+', ee_error_message)
     image_pixel = int(match[0])
@@ -53,3 +56,66 @@ def _square_roi(lon: float, lat: float, edge_size: int, scale: int) -> ee.Geomet
     half = edge_size * scale / 2
     point = ee.Geometry.Point([lon, lat])
     return point.buffer(half).bounds()
+
+
+
+def merge_tifs(
+    input_files: list[pathlib.Path],
+    output_path: pathlib.Path,
+    *,
+    nodata: int = 65535,
+    gdal_threads: int = 8
+) -> None:
+    """
+    Merge a list of GeoTIFF files into a single mosaic and write it out.
+
+    Parameters
+    ----------
+    input_files : list[Path]
+        Paths to the GeoTIFF tiles to be merged.
+    output_path : Path
+        Destination path for the merged GeoTIFF.
+    nodata : int, optional
+        NoData value to assign in the mosaic (default: 65535).
+    gdal_threads : int, optional
+        Number of GDAL threads to use for reading/writing (default: 8).
+
+    Raises
+    ------
+    ValueError
+        If `input_files` is empty.
+    """
+    if not input_files:
+        raise ValueError("The input_files list is empty")
+
+    # Ensure output path is a Path object
+    output_path = pathlib.Path(output_path).expanduser().resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Set GDAL threading environment
+    with rio.Env(GDAL_NUM_THREADS=str(gdal_threads), NUM_THREADS=str(gdal_threads)):
+        # Open all source datasets
+        srcs = [rio.open(fp) for fp in input_files]
+        try:
+            # Merge sources into one mosaic
+            mosaic, out_transform = merge(
+                srcs,
+                nodata=nodata,
+                resampling=Resampling.nearest
+            )
+
+            # Copy metadata from the first source and update it
+            meta = srcs[0].profile.copy()
+            meta.update({
+                "transform": out_transform,
+                "height": mosaic.shape[1],
+                "width": mosaic.shape[2]
+            })
+
+            # Write the merged mosaic to disk
+            with rio.open(output_path, "w", **meta) as dst:
+                dst.write(mosaic)
+        finally:
+            # Always close all open datasets
+            for src in srcs:
+                src.close()

cubexpress/geotyping.py

@@ -306,13 +306,17 @@ class RequestSet(BaseModel):
                             "crsCode": meta.raster_transform.crs,
                         },
                     },
+                    # "cs_cdf": int(meta.id.split("_")[-1]) / 100,
+                    # "date": meta.id.split("_")[0],
                     "outname": f"{meta.id}.tif",
                 }
+                
                 for index, meta in enumerate(self.requestset)
             ]
         )
 
 
+
     def _validate_dataframe_schema(self) -> None:
         """
         Checks that the `_dataframe` contains the required columns and that each column
@@ -367,21 +371,7 @@ class RequestSet(BaseModel):
                             f"Column '{col_name}' has an invalid type in row {i}. "
                             f"Expected {expected_type}, got {type(value)}"
                         )
-
-        # B) Validation of the `manifest` column structure
-        #    - Must contain at least 'assetId' or 'expression'
-        #    - Must contain 'grid' with the minimum required sub-keys
-        #    - Example:
-        #         {
-        #           "fileFormat": "GEO_TIFF",
-        #           "bandIds": [...],
-        #           "grid": {
-        #              "dimensions": {"width": ..., "height": ...},
-        #              "affineTransform": {...},
-        #              "crsCode": ...
-        #           },
-        #           // Either "assetId" or "expression" must be here
-        #         }
+                    
         for i, row in self._dataframe.iterrows():
             manifest = row["manifest"]
 

cubexpress/request.py

@@ -31,8 +31,7 @@ def table_to_requestset(
         If *df* is empty after filtering.
 
     """
-
-
+    
     df = table.copy()
 
     if df.empty:
@@ -48,33 +47,65 @@ def table_to_requestset(
     reqs: list[Request] = []
 
     if mosaic:
-        # group all asset IDs per day
+
         grouped = (
-            df.groupby("date")["id"]   # Series con listas de ids por día
-            .apply(list)
+            df.groupby('date')
+            .agg(
+                id_list     = ('id', list),
+                tiles       = (
+                    'id',
+                    lambda ids: ','.join(
+                        sorted({i.split('_')[-1][1:] for i in ids})
+                    )
+                ),
+                cs_cdf_mean = (
+                    'cs_cdf',
+                    lambda x: int(round(x.mean(), 2) * 100)
+                )
+            )
         )
 
-        for day, img_ids in grouped.items():
-            ee_img = ee.ImageCollection(
-                [ee.Image(f"{df.attrs['collection']}/{img}") for img in img_ids]
-            ).mosaic()
-
-            reqs.append(
-                Request(
-                    id=f"{day}_{centre_hash}",
-                    raster_transform=rt,
-                    image=ee_img,
-                    bands=df.attrs["bands"],
+        for day, row in grouped.iterrows():
+            
+            
+            img_ids   = row["id_list"]
+            cdf  = row["cs_cdf_mean"]
+            
+            if len(img_ids) > 1:
+
+                ee_img = ee.ImageCollection(
+                    [ee.Image(f"{df.attrs['collection']}/{img}") for img in img_ids]
+                ).mosaic()
+
+                reqs.append(
+                    Request(
+                        id=f"{day}_{centre_hash}_{cdf}",
+                        raster_transform=rt,
+                        image=ee_img,
+                        bands=df.attrs["bands"],
+                    )
                 )
-            )
-    else:  # one request per asset
+            else:
+                for img_id in img_ids:
+                    # tile = img_id.split("_")[-1][1:]
+                    reqs.append(
+                        Request(
+                            # id=f"{day}_{centre_hash}_{tile}_{cdf}",
+                            id=f"{day}_{centre_hash}_{cdf}",
+                            raster_transform=rt,
+                            image=f"{df.attrs['collection']}/{img_id}",
+                            bands=df.attrs["bands"],
+                        )
+                    )
+    else:
         for _, row in df.iterrows():
             img_id = row["id"]
-            day    = row["date"]
-
+            # tile = img_id.split("_")[-1][1:]
+            day = row["date"]
+            cdf = int(round(row["cs_cdf"], 2) * 100)
             reqs.append(
                 Request(
-                    id=f"{day}_{centre_hash}_{img_id}",
+                    id=f"{day}_{centre_hash}_{cdf}",
                     raster_transform=rt,
                     image=f"{df.attrs['collection']}/{img_id}",
                     bands=df.attrs["bands"],

{cubexpress-0.1.9.dist-info → cubexpress-0.1.11.dist-info}/METADATA

@@ -1,9 +1,8 @@
 Metadata-Version: 2.1
 Name: cubexpress
-Version: 0.1.9
+Version: 0.1.11
 Summary: Efficient processing of cubic Earth-observation (EO) data.
 Home-page: https://github.com/andesdatacube/cubexpress
-License: MIT
 Keywords: earth-engine,sentinel-2,geospatial,eo,cube
 Author: Julio Contreras
 Author-email: contrerasnetk@gmail.com
@@ -20,6 +19,7 @@ Requires-Dist: earthengine-api (>=1.5.12)
 Requires-Dist: numpy (>=2.0.2)
 Requires-Dist: pandas (>=2.2.2)
 Requires-Dist: pyarrow (>=14.0.0)
+Requires-Dist: pydantic (>=2.11.4)
 Requires-Dist: pygeohash (>=1.2.0)
 Requires-Dist: pyproj (>=3.6.0)
 Requires-Dist: rasterio (>=1.3.9)

cubexpress-0.1.11.dist-info/RECORD

@@ -0,0 +1,13 @@
+cubexpress/__init__.py,sha256=sKXcYQQPREFhVCHP81lL_5hAurUTm8MX1xVOEOMF-nA,618
+cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
+cubexpress/cloud_utils.py,sha256=BxS3HADLNj6rdFGYUjpcXA1Vvsa87JoL28YEAsu51H4,7482
+cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
+cubexpress/cube.py,sha256=SMN6MvezfeHipFE4v4f23dxWGk9h2t2s2aeeppD0voY,4133
+cubexpress/downloader.py,sha256=XsLDlq2ZHEccc1ET8ghnuOIYtGazVDwXohMSWBemVMw,4067
+cubexpress/geospatial.py,sha256=2DGwl3pyfNEOj8nn9gjc-tiiTXhV2ez9Bghz1I0vERs,3822
+cubexpress/geotyping.py,sha256=Fbnn7EoRvXrtjTRFTS4CPzQbxG4PA6WkfeM4YUp9iKg,16696
+cubexpress/request.py,sha256=PiDqnt3qB9tac4KkZdPIrv5VeRHqobk1u2q1VCCH2lI,3390
+cubexpress-0.1.11.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
+cubexpress-0.1.11.dist-info/METADATA,sha256=pxtqImmO_wIyA9P_0TWaxxps0O-95O6aVLbfEQ9GvBk,9651
+cubexpress-0.1.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+cubexpress-0.1.11.dist-info/RECORD,,

cubexpress-0.1.9.dist-info/RECORD

@@ -1,13 +0,0 @@
-cubexpress/__init__.py,sha256=RjyAqwiD0rU_Z5tCJTYNGKXZ1ggpfPB51wzhr0KwweY,570
-cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
-cubexpress/cloud_utils.py,sha256=aamTm-PxbPQ4ARwd5faG1a1sjKegbtkd0LxT7wYZJ60,7238
-cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
-cubexpress/cube.py,sha256=fwD_UdH0oBWSK-2-fMPPm3YKxcw1xxnm2g0vrZuChI8,3172
-cubexpress/downloader.py,sha256=NoJXxCZ7SXBMzUDcXU6DGa2vce61g716FYYfq17pH0k,4461
-cubexpress/geospatial.py,sha256=ZbsPIgsYQFnNFXUuQ136rJsL4b2Bf91o0Vsswby2dFc,1812
-cubexpress/geotyping.py,sha256=XuBcJAgNxvXCCIDmWijI70p6dEFlu6UfbqwQlWXSWQw,17155
-cubexpress/request.py,sha256=ZWVIXo0_rVkX1fBWREbtvvdYUSZPCv4LIcPdrMKKuLs,2270
-cubexpress-0.1.9.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
-cubexpress-0.1.9.dist-info/METADATA,sha256=qplHASBXni3m6kOAFIw8Jy2fBFqY1QfLDaNM3ou6cMk,9628
-cubexpress-0.1.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-cubexpress-0.1.9.dist-info/RECORD,,