satcube 0.1.13__tar.gz → 0.1.15__tar.gz

{satcube-0.1.13 → satcube-0.1.15}/PKG-INFO

@@ -1,29 +1,25 @@
 Metadata-Version: 2.1
 Name: satcube
-Version: 0.1.13
+Version: 0.1.15
 Summary: A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data.
 Home-page: https://github.com/IPL-UV/satcube
 Author: Cesar Aybar
 Author-email: fcesar.aybar@uv.es
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.9
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: full
-Requires-Dist: cubexpress (>=0.1.7)
-Requires-Dist: earthengine-api (>=1.5.12)
+Requires-Dist: cubexpress (>=0.1.10)
 Requires-Dist: mlstac (>=0.4.0)
-Requires-Dist: numpy (>=1.25.0)
-Requires-Dist: pandas (>=2.0.0)
 Requires-Dist: phicloudmask (>=0.0.2)
-Requires-Dist: pydantic (>=2.8.0)
-Requires-Dist: rasterio (>=1.3.9)
 Requires-Dist: requests (>=2.26.0)
-Requires-Dist: satalign (>=0.1.11)
+Requires-Dist: satalign (>=0.1.9)
 Requires-Dist: scikit-learn (>=1.2.0)
 Requires-Dist: segmentation-models-pytorch (>=0.3.0)
-Requires-Dist: utm (>=0.7.0)
+Requires-Dist: tqdm (>=4.67.1)
 Requires-Dist: xarray (>=2023.7.0)
 Project-URL: Documentation, https://ipl-uv.github.io/satcube/
 Project-URL: Repository, https://github.com/IPL-UV/satcube

{satcube-0.1.13 → satcube-0.1.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name    = "satcube"
-version = "0.1.13"
+version = "0.1.15"
 description = "A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data."
 authors = ["Cesar Aybar <fcesar.aybar@uv.es>"]
 repository     = "https://github.com/IPL-UV/satcube"
@@ -9,21 +9,16 @@ readme         = "README.md"
 packages       = [{ include = "satcube" }]
 
 [tool.poetry.dependencies]
-python          = ">=3.10,<4.0"
-cubexpress      = ">=0.1.7"
+python            = ">=3.9"
+cubexpress      = ">=0.1.10"
 mlstac          = ">=0.4.0"
-satalign        = ">=0.1.11"         
+satalign        = ">=0.1.9"         
 segmentation-models-pytorch = ">=0.3.0"
+phicloudmask    = ">=0.0.2" 
 scikit-learn    = ">=1.2.0"
-pandas          = ">=2.0.0"
-pydantic        = ">=2.8.0"
-rasterio          = ">=1.3.9"  
-earthengine-api   = ">=1.5.12"  
-numpy           = ">=1.25.0"
 requests        = ">=2.26.0"
-xarray          = ">=2023.7.0"
-utm             = ">=0.7.0"   
-phicloudmask    = ">=0.0.2" 
+xarray          = ">=2023.7.0"  
+tqdm            = ">=4.67.1"
 
 [tool.poetry.extras]
 full = ["torch"]

satcube-0.1.15/satcube/__init__.py

@@ -0,0 +1,9 @@
+from satcube.cloud_detection import cloud_masking
+from satcube.download import download
+from satcube.align import align
+import importlib.metadata
+from satcube.objects  import SatCubeMetadata  
+
+__all__ = ["cloud_masking", "download", "align", "SatCubeMetadata"]
+# __version__ = importlib.metadata.version("satcube")
+

satcube-0.1.15/satcube/align.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import pathlib
+from typing import List, Tuple
+import pickle
+import pandas as pd
+import satalign
+import shutil
+
+import numpy as np
+import rasterio as rio
+import xarray as xr
+from affine import Affine
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+
+def process_row(row: pd.Series, reference: np.ndarray, input_dir: pathlib.Path, output_dir: pathlib.Path) -> None:
+    row_path = input_dir / (row["id"] + ".tif")
+    output_path = output_dir / (row["id"] + ".tif")
+    with rio.open(row_path) as src:
+        row_image = src.read()
+        profile_image = src.profile
+    
+    row_image_float = row_image.astype(np.float32) / 10000
+    row_image_float = row_image_float[np.newaxis, ...]
+    
+    pcc_model = satalign.LGM(
+        datacube     = row_image_float,
+        reference    = reference
+    )
+    image, _ = pcc_model.run_multicore()
+    image = (image * 10000).astype(np.uint16).squeeze()
+
+    with rio.open(output_path, "w", **profile_image) as dst:
+        dst.write(image)
+
+def align(
+    input_dir: str | pathlib.Path = "raw",
+    output_dir: str | pathlib.Path = "aligned",
+    nworks: int = 4,
+    cache: bool = False
+) -> None:
+    
+    input_dir = pathlib.Path(input_dir).expanduser().resolve()
+    output_dir = pathlib.Path(output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    metadata_path = input_dir / "metadata.csv"
+    
+    if not metadata_path.exists():
+        raise FileNotFoundError(
+            f"Metadata file not found: {metadata_path}. "
+            "Please run the download step first."
+        )
+    else:
+        metadata = pd.read_csv(metadata_path)
+        
+    if cache:
+        exist_files = [file.stem for file in output_dir.glob("*.tif")]
+        metadata = metadata[~metadata["id"].isin(exist_files)]
+        
+        if metadata.empty:
+            return
+
+    id_reference = metadata.sort_values(
+        by=["cs_cdf", "date"],
+        ascending=False,
+    ).iloc[0]["id"]
+    
+    reference_path = input_dir / (id_reference + ".tif")
+    
+    with rio.open(reference_path) as ref_src:
+        reference = ref_src.read()
+        
+    reference_float = reference.astype(np.float32) / 10000
+
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = {
+            executor.submit(process_row, row, reference_float, input_dir, output_dir)
+            for _, row in metadata.iterrows()
+        }
+        for future in tqdm(
+            as_completed(futures), 
+            total=len(futures), 
+            desc="Aligning images",
+            unit="image",
+            leave=True
+        ):
+            try:
+                future.result()
+            except Exception as e:
+                print(f"Error processing image: {e}")
+                
+    metadata = input_dir / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = output_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

satcube-0.1.15/satcube/cloud_detection.py

@@ -0,0 +1,238 @@
+"""Predict cloud masks for Sentinel-2 GeoTIFFs with the SEN2CloudEnsemble model.
+
+The callable :pyfunc:`cloud_masking` accepts **either** a single ``.tif`` file  
+or a directory tree; in both cases it writes a masked copy of every image (and,
+optionally, the binary mask) to *output*.
+
+Example
+-------
+>>> from satcube.cloud_detection import cloud_masking
+>>> cloud_masking("~/s2/input", "~/s2/output", device="cuda")
+"""
+
+from __future__ import annotations
+
+import pathlib
+
+import mlstac
+import numpy as np
+import rasterio as rio
+from rasterio.windows import Window
+import torch
+import pandas as pd
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+import rasterio as rio
+from rasterio.merge import merge
+import shutil
+
+from satcube.utils import define_iteration, DeviceManager
+import warnings
+warnings.filterwarnings(
+    "ignore",
+    message="The secret HF_TOKEN does not exist in your Colab secrets.",
+    category=UserWarning,
+    module=r"huggingface_hub\.utils\._.*",
+)
+
+
+
+def infer_cloudmask(
+    input_path: str | pathlib.Path,
+    output_path: str | pathlib.Path,
+    cloud_model: torch.nn.Module,
+    *,
+    chunk_size: int = 512,
+    overlap: int = 32,
+    device: str = "cpu",
+    save_mask: bool = False,
+    prefix: str = ""          
+) -> pathlib.Path:
+    """
+    Predict 'image_path' in overlapping patches of 'chunk_size' x 'chunk_size',
+    but only write the valid (inner) region to avoid seam artifacts.
+
+    This uses partial overlap logic:
+      - For interior tiles, skip overlap//2 on each side.
+      - For boundary tiles, we skip only the interior side to avoid losing data at the edges.
+
+    Parameters
+    ----------
+    image_path : Path to input image.
+    output_path : Path to output single-band mask.
+    cloud_model : PyTorch model (already loaded with weights).
+    chunk_size : Size of each tile to read from the source image (default 512).
+    overlap : Overlap in pixels between adjacent tiles (default 32).
+    device : "cpu" or "cuda:0".
+
+    Returns
+    -------
+    pathlib.Path : The path to the created output image.
+    """
+
+    input_path = pathlib.Path(input_path)
+    output_path = pathlib.Path(output_path)
+
+    with rio.open(input_path) as src:
+        meta = src.profile
+        if not meta.get("tiled", False):
+            raise ValueError("The input image is not marked as tiled in its metadata.")
+        # Ensure the internal blocksize matches chunk_size
+        if chunk_size % meta["blockxsize"] != 0 and meta["blockxsize"] <= chunk_size:
+            raise ValueError(f"Image blocks must be {chunk_size}x{chunk_size}, "
+                            f"got {meta['blockxsize']}x{meta['blockysize']}")
+        height, width = meta["height"], meta["width"]
+
+    full_mask = np.zeros((height, width), dtype=np.float32)
+
+    coords = define_iteration((height, width), chunk_size, overlap)
+
+    with rio.open(input_path) as src:
+
+        for (row_off, col_off) in coords: 
+
+            window = Window(col_off, row_off, chunk_size, chunk_size)
+            patch = src.read(window=window) / 1e4
+            patch_tensor = torch.from_numpy(patch).float().unsqueeze(0).to(device)
+            result = cloud_model(patch_tensor).cpu().numpy().astype(np.uint8)
+      
+            if col_off == 0:
+                offset_x = 0
+            else:
+                offset_x = col_off + overlap // 2
+            if row_off == 0:
+                offset_y = 0
+            else:
+                offset_y = row_off + overlap // 2
+            if (offset_x + chunk_size) == width:
+                length_x = chunk_size
+                sub_x_start = 0
+            else:
+                length_x = chunk_size - (overlap // 2)
+                sub_x_start = overlap // 2 if col_off != 0 else 0
+
+            if (offset_y + chunk_size) == height:
+                length_y = chunk_size
+                sub_y_start = 0
+            else:
+                length_y = chunk_size - (overlap // 2)
+                sub_y_start = overlap // 2 if row_off != 0 else 0
+
+            full_mask[
+                offset_y : offset_y + length_y,
+                offset_x : offset_x + length_x
+            ] = result[
+                sub_y_start : sub_y_start + length_y,
+                sub_x_start : sub_x_start + length_x
+            ]
+
+        if save_mask:
+            out_meta = meta.copy()
+            out_meta.update(count=1, dtype="uint8", nodata=255)
+            output_mask = output_path.parent / (output_path.stem + "_mask.tif")
+            with rio.open(output_mask, "w", **out_meta) as dst:
+                dst.write(full_mask, 1)
+            
+
+        data = src.read()
+        img_prof = src.profile.copy()
+
+        masked = data.copy()
+        masked[:, full_mask != 0] = 65535
+        img_prof.update(dtype="uint16", nodata=65535)
+
+        with rio.open(output_path, "w", **img_prof) as dst:
+            dst.write(masked)
+
+    return output_path
+
+def cloud_masking( 
+    input: str | pathlib.Path = "raw",
+    output: str | pathlib.Path = "masked",
+    model_path: str | pathlib.Path = "SEN2CloudEnsemble",
+    device: str = "cpu",
+    save_mask: bool = False,
+    nworks: int = 4,
+) -> list[pathlib.Path]:
+    """Write cloud-masked Sentinel-2 images.
+
+    Parameters
+    ----------
+    input
+        Path to a single ``.tif`` file **or** a directory containing them.
+    output
+        Destination directory (created i
+        f missing).
+    tile, pad
+        Tile size and padding (pixels) when tiling is required.
+    save_mask
+        If *True*, store the binary mask alongside the masked image.
+    device
+        Torch device for inference, e.g. ``"cpu"`` or ``"cuda:0"``.
+    max_pix_cpu
+        Tile images larger than this when running on CPU.
+
+    Returns
+    ------
+    list[pathlib.Path]
+        Paths to the generated masked images.
+    """
+    src = pathlib.Path(input).expanduser().resolve()
+    dst_dir = pathlib.Path(output).expanduser().resolve()
+    dst_dir.mkdir(parents=True, exist_ok=True)
+
+    # Collect files to process -------------------------------------------------
+    tif_paths = []
+    if src.is_dir():
+        tif_paths = [p for p in src.rglob("*.tif")]
+    elif src.is_file() and src.suffix.lower() == ".tif":
+        tif_paths = [src]
+        src = src.parent  # for relative-path bookkeeping below
+    else:
+        raise ValueError(f"Input must be a .tif or directory, got: {src}")
+
+    if not tif_paths:
+        print(f"[cloud_masking] No .tif files found in {src}")
+        return []
+    
+    if not pathlib.Path(model_path).exists():
+        mlstac.download(
+            file = "https://huggingface.co/tacofoundation/CloudSEN12-models/resolve/main/SEN2CloudEnsemble/mlm.json",
+            output_dir = model_path
+        )
+
+    model = mlstac.load(model_path)
+    cloud_model = DeviceManager(model, init_device=device).model
+    cloud_model.eval()
+    
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = { 
+            executor.submit(
+                infer_cloudmask,
+                input_path=p,
+                output_path=dst_dir / p.name,
+                cloud_model=cloud_model,
+                device=device,
+                save_mask=save_mask,
+                prefix=f"[{i+1}/{len(tif_paths)}] "
+            ): p for i, p in enumerate(tif_paths)
+        }
+        
+        for future in tqdm(
+            as_completed(futures),
+            total=len(futures),
+            desc="Cloud Masking",
+            position=0,
+            leave=True
+        ):
+            p = futures[future]
+            try:
+                result = future.result()
+                print(f"{result} processed successfully.")
+            except Exception as e:
+                print(f"Error processing {p}: {e}")
+                
+    metadata = src / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = dst_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

satcube-0.1.15/satcube/download.py

@@ -0,0 +1,68 @@
+import sys, time, threading, itertools
+import cubexpress as ce
+import pandas as pd
+from satcube.objects import SatCubeMetadata
+import pathlib
+
+def download(
+    lon: float,
+    lat: float,
+    edge_size: int,
+    start: str,
+    end: str,
+    *,
+    max_cscore: float = 1,
+    min_cscore: float = 0,
+    outfolder: str = "raw",
+    nworks: int = 4
+) -> "SatCubeMetadata":
+        
+        
+    outfolder = pathlib.Path(outfolder).resolve()
+
+    table = ce.s2_table(
+        lon=lon,
+        lat=lat,
+        edge_size=edge_size,
+        start=start,
+        end=end,
+        max_cscore=max_cscore,
+        min_cscore=min_cscore
+    )   
+        
+    requests = ce.table_to_requestset(
+        table=table,
+        mosaic=True
+    )
+    
+    ce.get_cube(
+        requests=requests,
+        outfolder=outfolder,
+        nworks=nworks
+    )
+
+    table_req = (
+        requests._dataframe.copy()
+        .drop(columns=['geotransform', 'manifest', 'outname', 'width', 'height', 'scale_x', 'scale_y'])
+    )
+    
+    table_req['date'] = table_req['id'].str.split('_').str[0]
+
+    result_table = (
+        table.groupby('date')
+        .agg(
+            id=('id', lambda x: '-'.join(x)),
+            cs_cdf=('cs_cdf', 'first')
+        )
+        .reset_index()  
+    )
+    
+    table_final = table_req.merge(
+        result_table,
+        on='date',
+        how='left'
+    ).rename(columns={'id_x': 'id', 'id_y': 'gee_ids'})
+
+    table_final.to_csv(outfolder / "metadata.csv", index=False)
+
+    return SatCubeMetadata(df=table_final, raw_dir=outfolder)

satcube-0.1.15/satcube/objects.py

@@ -0,0 +1,71 @@
+# satcube/objects.py
+from __future__ import annotations
+from dataclasses import dataclass, field
+import pathlib
+import pandas as pd
+
+from satcube.align  import align as _align_fn
+from satcube.cloud_detection  import cloud_masking as _cloud_fn
+
+@dataclass
+class SatCubeMetadata:
+    df: pd.DataFrame = field(repr=False)
+    raw_dir: pathlib.Path = field(repr=False)
+
+    def __repr__(self) -> str: 
+        return self.df.__repr__()
+    
+    __str__ = __repr__
+    
+    def _repr_html_(self) -> str:
+        html = getattr(self.df, "_repr_html_", None)
+        return html() if callable(html) else self.df.__repr__()
+
+    def align(
+        self,
+        input_dir: str | pathlib.Path | None = None,
+        output_dir: str | pathlib.Path = "aligned",
+        nworks: int = 4,
+        cache: bool = False
+    ) -> "SatCubeMetadata":
+        
+        if input_dir is None:
+            input_dir = self.raw_dir
+        else:
+            input_dir = pathlib.Path(input_dir).expanduser().resolve()
+
+        _align_fn(
+            input_dir=input_dir, 
+            output_dir=output_dir, 
+            nworks=nworks, 
+            cache=cache
+        )
+        self.aligned_dir = pathlib.Path(output_dir).resolve()
+        return self
+
+    def cloud_masking(
+        self,
+        output_dir: str | pathlib.Path = "masked",
+        model_path: str | pathlib.Path = "SEN2CloudEnsemble",
+        device: str = "cpu"
+    ) -> "SatCubeMetadata":
+        if not hasattr(self, "aligned_dir"):
+            raise RuntimeError("You must run .align() first")
+        _cloud_fn(
+            input=self.aligned_dir,
+            output=output_dir,
+            model_path=model_path,
+            device=device
+        )
+        self.masked_dir = pathlib.Path(output_dir).resolve()
+        return self
+    
+    def __getattr__(self, item):
+        return getattr(self.df, item)
+
+    def __getitem__(self, key):
+        return self.df.__getitem__(key)
+    
+    def __len__(self):
+        return len(self.df)
+    

{satcube-0.1.13 → satcube-0.1.15}/satcube/utils.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
 import gc
+import itertools
 from typing import Any, Optional
-
 import torch
 
-
 def _reset_gpu() -> None:
     """Release CUDA memory and reset allocation statistics.
 
@@ -15,6 +14,65 @@ def _reset_gpu() -> None:
     torch.cuda.reset_peak_memory_stats()
 
 
+def define_iteration(dimension: tuple, chunk_size: int, overlap: int = 0):
+    """
+    Define the iteration strategy to walk through the image with an overlap.
+
+    Args:
+        dimension (tuple): Dimension of the S2 image.
+        chunk_size (int): Size of the chunks.
+        overlap (int): Size of the overlap between chunks.
+
+    Returns:
+        list: List of chunk coordinates.
+    """
+    dimy, dimx = dimension
+
+    if chunk_size > max(dimx, dimy):
+        return [(0, 0)]
+
+    # Adjust step to create overlap
+    y_step = chunk_size - overlap
+    x_step = chunk_size - overlap
+
+    # Generate initial chunk positions
+    iterchunks = list(itertools.product(range(0, dimy, y_step), range(0, dimx, x_step)))
+
+    # Fix chunks at the edges to stay within bounds
+    iterchunks_fixed = fix_lastchunk(
+        iterchunks=iterchunks, s2dim=dimension, chunk_size=chunk_size
+    )
+
+    return iterchunks_fixed
+
+
+def fix_lastchunk(iterchunks, s2dim, chunk_size):
+    """
+    Fix the last chunk of the overlay to ensure it aligns with image boundaries.
+
+    Args:
+        iterchunks (list): List of chunks created by itertools.product.
+        s2dim (tuple): Dimension of the S2 images.
+        chunk_size (int): Size of the chunks.
+
+    Returns:
+        list: List of adjusted chunk coordinates.
+    """
+    itercontainer = []
+
+    for index_i, index_j in iterchunks:
+        # Adjust if the chunk extends beyond bounds
+        if index_i + chunk_size > s2dim[0]:
+            index_i = max(s2dim[0] - chunk_size, 0)
+        if index_j + chunk_size > s2dim[1]:
+            index_j = max(s2dim[1] - chunk_size, 0)
+
+        itercontainer.append((index_i, index_j))
+
+    return itercontainer
+
+
+
 class DeviceManager:
     """Hold a compiled mlstac model and move it between devices on demand."""
 
@@ -68,3 +126,4 @@ class DeviceManager:
         self.model = self._experiment.compiled_model(device=new_device, mode="max")
         self.device = new_device
         return self.model
+

satcube-0.1.13/satcube/__init__.py

@@ -1,10 +0,0 @@
-from satcube.cloud_detection import cloud_masking
-from satcube.download import download_data
-
-
-
-__all__ = ["cloud_masking", "download_data"]
-
-import importlib.metadata
-__version__ = importlib.metadata.version("satcube")
-