satcube 0.1.14__tar.gz → 0.1.15__tar.gz

{satcube-0.1.14 → satcube-0.1.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: satcube
-Version: 0.1.14
+Version: 0.1.15
 Summary: A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data.
 Home-page: https://github.com/IPL-UV/satcube
 Author: Cesar Aybar
@@ -16,9 +16,10 @@ Requires-Dist: cubexpress (>=0.1.10)
 Requires-Dist: mlstac (>=0.4.0)
 Requires-Dist: phicloudmask (>=0.0.2)
 Requires-Dist: requests (>=2.26.0)
-Requires-Dist: satalign (>=0.1.12)
+Requires-Dist: satalign (>=0.1.9)
 Requires-Dist: scikit-learn (>=1.2.0)
 Requires-Dist: segmentation-models-pytorch (>=0.3.0)
+Requires-Dist: tqdm (>=4.67.1)
 Requires-Dist: xarray (>=2023.7.0)
 Project-URL: Documentation, https://ipl-uv.github.io/satcube/
 Project-URL: Repository, https://github.com/IPL-UV/satcube

{satcube-0.1.14 → satcube-0.1.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name    = "satcube"
-version = "0.1.14"
+version = "0.1.15"
 description = "A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data."
 authors = ["Cesar Aybar <fcesar.aybar@uv.es>"]
 repository     = "https://github.com/IPL-UV/satcube"
@@ -12,12 +12,13 @@ packages       = [{ include = "satcube" }]
 python            = ">=3.9"
 cubexpress      = ">=0.1.10"
 mlstac          = ">=0.4.0"
-satalign        = ">=0.1.12"         
+satalign        = ">=0.1.9"         
 segmentation-models-pytorch = ">=0.3.0"
 phicloudmask    = ">=0.0.2" 
 scikit-learn    = ">=1.2.0"
 requests        = ">=2.26.0"
 xarray          = ">=2023.7.0"  
+tqdm            = ">=4.67.1"
 
 [tool.poetry.extras]
 full = ["torch"]

satcube-0.1.15/satcube/__init__.py

@@ -0,0 +1,9 @@
+from satcube.cloud_detection import cloud_masking
+from satcube.download import download
+from satcube.align import align
+import importlib.metadata
+from satcube.objects  import SatCubeMetadata  
+
+__all__ = ["cloud_masking", "download", "align", "SatCubeMetadata"]
+# __version__ = importlib.metadata.version("satcube")
+

satcube-0.1.15/satcube/align.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import pathlib
+from typing import List, Tuple
+import pickle
+import pandas as pd
+import satalign
+import shutil
+
+import numpy as np
+import rasterio as rio
+import xarray as xr
+from affine import Affine
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+
+def process_row(row: pd.Series, reference: np.ndarray, input_dir: pathlib.Path, output_dir: pathlib.Path) -> None:
+    row_path = input_dir / (row["id"] + ".tif")
+    output_path = output_dir / (row["id"] + ".tif")
+    with rio.open(row_path) as src:
+        row_image = src.read()
+        profile_image = src.profile
+    
+    row_image_float = row_image.astype(np.float32) / 10000
+    row_image_float = row_image_float[np.newaxis, ...]
+    
+    pcc_model = satalign.LGM(
+        datacube     = row_image_float,
+        reference    = reference
+    )
+    image, _ = pcc_model.run_multicore()
+    image = (image * 10000).astype(np.uint16).squeeze()
+
+    with rio.open(output_path, "w", **profile_image) as dst:
+        dst.write(image)
+
+def align(
+    input_dir: str | pathlib.Path = "raw",
+    output_dir: str | pathlib.Path = "aligned",
+    nworks: int = 4,
+    cache: bool = False
+) -> None:
+    
+    input_dir = pathlib.Path(input_dir).expanduser().resolve()
+    output_dir = pathlib.Path(output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    metadata_path = input_dir / "metadata.csv"
+    
+    if not metadata_path.exists():
+        raise FileNotFoundError(
+            f"Metadata file not found: {metadata_path}. "
+            "Please run the download step first."
+        )
+    else:
+        metadata = pd.read_csv(metadata_path)
+        
+    if cache:
+        exist_files = [file.stem for file in output_dir.glob("*.tif")]
+        metadata = metadata[~metadata["id"].isin(exist_files)]
+        
+        if metadata.empty:
+            return
+
+    id_reference = metadata.sort_values(
+        by=["cs_cdf", "date"],
+        ascending=False,
+    ).iloc[0]["id"]
+    
+    reference_path = input_dir / (id_reference + ".tif")
+    
+    with rio.open(reference_path) as ref_src:
+        reference = ref_src.read()
+        
+    reference_float = reference.astype(np.float32) / 10000
+
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = {
+            executor.submit(process_row, row, reference_float, input_dir, output_dir)
+            for _, row in metadata.iterrows()
+        }
+        for future in tqdm(
+            as_completed(futures), 
+            total=len(futures), 
+            desc="Aligning images",
+            unit="image",
+            leave=True
+        ):
+            try:
+                future.result()
+            except Exception as e:
+                print(f"Error processing image: {e}")
+                
+    metadata = input_dir / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = output_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

{satcube-0.1.14 → satcube-0.1.15}/satcube/cloud_detection.py

@@ -20,9 +20,11 @@ import rasterio as rio
 from rasterio.windows import Window
 import torch
 import pandas as pd
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 import rasterio as rio
 from rasterio.merge import merge
+import shutil
 
 from satcube.utils import define_iteration, DeviceManager
 import warnings
@@ -35,7 +37,6 @@ warnings.filterwarnings(
 
 
 
-
 def infer_cloudmask(
     input_path: str | pathlib.Path,
     output_path: str | pathlib.Path,
@@ -68,13 +69,10 @@ def infer_cloudmask(
     -------
     pathlib.Path : The path to the created output image.
     """
-    # image_path = "/home/contreras/Documents/GitHub/satcube2/raw/2018-07-15_6q49m.tif"
-    # output_path = "/home/contreras/Documents/GitHub/satcube2/masked/2018-07-15_6q49m.tif"
-    
+
     input_path = pathlib.Path(input_path)
     output_path = pathlib.Path(output_path)
 
-    # 1) Validate metadata
     with rio.open(input_path) as src:
         meta = src.profile
         if not meta.get("tiled", False):
@@ -85,21 +83,13 @@ def infer_cloudmask(
                             f"got {meta['blockxsize']}x{meta['blockysize']}")
         height, width = meta["height"], meta["width"]
 
-
-    # 2) Crear un buffer en RAM (float32)
     full_mask = np.zeros((height, width), dtype=np.float32)
 
-    # 3) Iterar por ventanas
-
     coords = define_iteration((height, width), chunk_size, overlap)
 
     with rio.open(input_path) as src:
 
-        for (row_off, col_off) in tqdm(
-            coords, 
-            desc=f"{prefix} Inference on {input_path.name}",
-            position=1,
-            leave=False):
+        for (row_off, col_off) in coords: 
 
             window = Window(col_off, row_off, chunk_size, chunk_size)
             patch = src.read(window=window) / 1e4
@@ -143,9 +133,9 @@ def infer_cloudmask(
             with rio.open(output_mask, "w", **out_meta) as dst:
                 dst.write(full_mask, 1)
             
-        with rio.open(input_path) as src_img:
-            data = src_img.read()
-            img_prof = src_img.profile.copy()
+
+        data = src.read()
+        img_prof = src.profile.copy()
 
         masked = data.copy()
         masked[:, full_mask != 0] = 65535
@@ -160,9 +150,9 @@ def cloud_masking(
     input: str | pathlib.Path = "raw",
     output: str | pathlib.Path = "masked",
     model_path: str | pathlib.Path = "SEN2CloudEnsemble",
-    save_mask: bool = False,
     device: str = "cpu",
-    cache: bool = True
+    save_mask: bool = False,
+    nworks: int = 4,
 ) -> list[pathlib.Path]:
     """Write cloud-masked Sentinel-2 images.
 
@@ -214,29 +204,35 @@ def cloud_masking(
     model = mlstac.load(model_path)
     cloud_model = DeviceManager(model, init_device=device).model
     cloud_model.eval()
-
-    outs = [
-        infer_cloudmask(
-            input_path=p,
-            output_path=dst_dir / p.name,
-            cloud_model=cloud_model,
-            device=device,
-            save_mask=save_mask,
-            prefix=f"[{i+1}/{len(tif_paths)}] "
-        )
-        for i, p in enumerate(tif_paths)
-    ]
-
-    df_out = (
-        pd.Series(outs, name="path")
-        .to_frame()
-        .assign(path_str=lambda df: df["path"].astype(str))
-        .assign(
-            date  = lambda df: df["path_str"].str.extract(r"(\d{4}-\d{2}-\d{2})", expand=False),
-            score = lambda df: df["path_str"].str.extract(r"_(\d+)\.tif$", expand=False).astype(int) / 100
-        )
-        .drop(columns="path_str")                                    # ya no la necesitamos
-        .sort_values("score", ascending=False, ignore_index=True)
-    )
-
-    return df_out
+    
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = { 
+            executor.submit(
+                infer_cloudmask,
+                input_path=p,
+                output_path=dst_dir / p.name,
+                cloud_model=cloud_model,
+                device=device,
+                save_mask=save_mask,
+                prefix=f"[{i+1}/{len(tif_paths)}] "
+            ): p for i, p in enumerate(tif_paths)
+        }
+        
+        for future in tqdm(
+            as_completed(futures),
+            total=len(futures),
+            desc="Cloud Masking",
+            position=0,
+            leave=True
+        ):
+            p = futures[future]
+            try:
+                result = future.result()
+                print(f"{result} processed successfully.")
+            except Exception as e:
+                print(f"Error processing {p}: {e}")
+                
+    metadata = src / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = dst_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

satcube-0.1.15/satcube/download.py

@@ -0,0 +1,68 @@
+import sys, time, threading, itertools
+import cubexpress as ce
+import pandas as pd
+from satcube.objects import SatCubeMetadata
+import pathlib
+
+def download(
+    lon: float,
+    lat: float,
+    edge_size: int,
+    start: str,
+    end: str,
+    *,
+    max_cscore: float = 1,
+    min_cscore: float = 0,
+    outfolder: str = "raw",
+    nworks: int = 4
+) -> "SatCubeMetadata":
+        
+        
+    outfolder = pathlib.Path(outfolder).resolve()
+
+    table = ce.s2_table(
+        lon=lon,
+        lat=lat,
+        edge_size=edge_size,
+        start=start,
+        end=end,
+        max_cscore=max_cscore,
+        min_cscore=min_cscore
+    )   
+        
+    requests = ce.table_to_requestset(
+        table=table,
+        mosaic=True
+    )
+    
+    ce.get_cube(
+        requests=requests,
+        outfolder=outfolder,
+        nworks=nworks
+    )
+
+    table_req = (
+        requests._dataframe.copy()
+        .drop(columns=['geotransform', 'manifest', 'outname', 'width', 'height', 'scale_x', 'scale_y'])
+    )
+    
+    table_req['date'] = table_req['id'].str.split('_').str[0]
+
+    result_table = (
+        table.groupby('date')
+        .agg(
+            id=('id', lambda x: '-'.join(x)),
+            cs_cdf=('cs_cdf', 'first')
+        )
+        .reset_index()  
+    )
+    
+    table_final = table_req.merge(
+        result_table,
+        on='date',
+        how='left'
+    ).rename(columns={'id_x': 'id', 'id_y': 'gee_ids'})
+
+    table_final.to_csv(outfolder / "metadata.csv", index=False)
+
+    return SatCubeMetadata(df=table_final, raw_dir=outfolder)

satcube-0.1.15/satcube/objects.py

@@ -0,0 +1,71 @@
+# satcube/objects.py
+from __future__ import annotations
+from dataclasses import dataclass, field
+import pathlib
+import pandas as pd
+
+from satcube.align  import align as _align_fn
+from satcube.cloud_detection  import cloud_masking as _cloud_fn
+
+@dataclass
+class SatCubeMetadata:
+    df: pd.DataFrame = field(repr=False)
+    raw_dir: pathlib.Path = field(repr=False)
+
+    def __repr__(self) -> str: 
+        return self.df.__repr__()
+    
+    __str__ = __repr__
+    
+    def _repr_html_(self) -> str:
+        html = getattr(self.df, "_repr_html_", None)
+        return html() if callable(html) else self.df.__repr__()
+
+    def align(
+        self,
+        input_dir: str | pathlib.Path | None = None,
+        output_dir: str | pathlib.Path = "aligned",
+        nworks: int = 4,
+        cache: bool = False
+    ) -> "SatCubeMetadata":
+        
+        if input_dir is None:
+            input_dir = self.raw_dir
+        else:
+            input_dir = pathlib.Path(input_dir).expanduser().resolve()
+
+        _align_fn(
+            input_dir=input_dir, 
+            output_dir=output_dir, 
+            nworks=nworks, 
+            cache=cache
+        )
+        self.aligned_dir = pathlib.Path(output_dir).resolve()
+        return self
+
+    def cloud_masking(
+        self,
+        output_dir: str | pathlib.Path = "masked",
+        model_path: str | pathlib.Path = "SEN2CloudEnsemble",
+        device: str = "cpu"
+    ) -> "SatCubeMetadata":
+        if not hasattr(self, "aligned_dir"):
+            raise RuntimeError("You must run .align() first")
+        _cloud_fn(
+            input=self.aligned_dir,
+            output=output_dir,
+            model_path=model_path,
+            device=device
+        )
+        self.masked_dir = pathlib.Path(output_dir).resolve()
+        return self
+    
+    def __getattr__(self, item):
+        return getattr(self.df, item)
+
+    def __getitem__(self, key):
+        return self.df.__getitem__(key)
+    
+    def __len__(self):
+        return len(self.df)
+    

satcube-0.1.14/satcube/__init__.py

@@ -1,10 +0,0 @@
-from satcube.cloud_detection import cloud_masking
-from satcube.download import download
-from satcube.align import align
-
-
-__all__ = ["cloud_masking", "download", "align"]
-
-import importlib.metadata
-__version__ = importlib.metadata.version("satcube")
-

satcube-0.1.14/satcube/align.py

@@ -1,149 +0,0 @@
-from __future__ import annotations
-
-import pathlib
-from typing import List, Tuple
-import pickle
-
-import numpy as np
-import rasterio
-import xarray as xr
-from affine import Affine
-
-import satalign as sat
-
-
-def align(
-    input_dir: str | pathlib.Path,
-    output_dir: str | pathlib.Path,
-    *,
-    channel: str = "mean",
-    crop_center: int = 128,
-    num_threads: int = 2,
-    save_tiffs: bool = True,
-) -> Tuple[xr.DataArray, List[np.ndarray]]:
-    """Align all masked Sentinel‑2 tiles found in *input_dir*.
-
-    Parameters
-    ----------
-    input_dir
-        Directory containing masked Sentinel‑2 *TIFF* tiles produced by the
-        previous preprocessing stage.
-    output_dir
-        Directory where the alignment artefacts (``datacube.pickle``) and, if
-        requested, one aligned *GeoTIFF* per date will be written.
-    channel
-        Datacube band used by the *PCC* model for correlation.  ``"mean"`` is
-        recommended because it carries fewer noise artefacts.
-    crop_center
-        Half‑size (in pixels) of the square window extracted around the scene
-        centre that is fed to the correlation engine.
-    num_threads
-        Number of CPU threads for the multi‑core phase‑correlation run.
-    save_tiffs
-        If *True* (default) the aligned datacube is exported to tiled *COGs* via
-        :func:`save_aligned_cube_to_tiffs`.
-    pickle_datacube
-        If *True* (default) the raw (unaligned) datacube is pickled to
-        ``datacube.pickle`` inside *output_dir* for reproducibility/debugging.
-
-    Returns
-    -------
-    aligned_cube, warp_matrices
-        *aligned_cube* is the spatially aligned datacube as an
-        :class:`xarray.DataArray`; *warp_matrices* is the list of 3 × 3 affine
-        homography matrices (one per time step) returned by the
-        :pyclass:`~satalign.PCC` engine.
-    """
-    input_path = pathlib.Path(input_dir).expanduser().resolve()
-    output_path = pathlib.Path(output_dir).expanduser().resolve()
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    # ── 1. Build datacube ────────────────────────────────────────────
-    da =  sat.utils.create_array(input_path)
-
-    # ── 2. Select reference slice (highest cloud‑score CDF) ───────────
-    da_sorted = da.sortby("cs_cdf", ascending=False)
-    ref_slice = da_sorted.isel(time=0)
-    reference = ref_slice.where((ref_slice != 0) & (ref_slice != 65535))
-
-    # ── 3. Instantiate and run PCC model ─────────────────────────────
-    pcc_model = sat.PCC(
-        datacube=da,
-        reference=reference,
-        channel=channel,
-        crop_center=crop_center,
-        num_threads=num_threads,
-    )
-    aligned_cube, warp_matrices = pcc_model.run_multicore()
-
-    # ── 4. Optionally export as Cloud‑Optimised GeoTIFFs ────────────
-    if save_tiffs:
-        save_aligned_cube_to_tiffs(aligned_cube, output_path)
-
-    return aligned_cube, warp_matrices
-
-
-def save_aligned_cube_to_tiffs(
-    aligned_cube: xr.DataArray,
-    out_dir: str | pathlib.Path,
-    *,
-    block_size: int = 128,
-) -> None:
-    """Write each time slice of *aligned_cube* to an individual tiled COG.
-
-    The filenames follow the pattern ``YYYY‑MM‑DD.tif``.
-
-    Parameters
-    ----------
-    aligned_cube
-        Datacube returned by :func:`align_datacube`.
-    out_dir
-        Target directory; it will be created if it does not exist.
-    block_size
-        Internal tile size (*rasterio* ``blockxsize`` and ``blockysize``).
-    """
-    out_dir_path = pathlib.Path(out_dir).expanduser().resolve()
-    out_dir_path.mkdir(parents=True, exist_ok=True)
-
-    # ── 1. Build affine transform from x/y coordinate vectors ────────
-    x_vals = aligned_cube.x.values
-    y_vals = aligned_cube.y.values
-    x_res = float(x_vals[1] - x_vals[0])  # positive (east)
-    y_res = float(y_vals[1] - y_vals[0])  # negative (north‑up)
-    transform = (
-        Affine.translation(x_vals[0] - x_res / 2.0, y_vals[0] - y_res / 2.0)
-        * Affine.scale(x_res, y_res)
-    )
-
-    # ── 2. Retrieve CRS from datacube attributes ────────────────────
-    attrs = aligned_cube.attrs
-    crs: str | None = attrs.get("crs_wkt")
-    if crs is None and "crs_epsg" in attrs:
-        crs = f"EPSG:{int(attrs['crs_epsg'])}"
-
-    # ── 3. Loop over acquisition dates and write GeoTIFFs ────────────
-    for t in aligned_cube.time.values:
-        date_str = str(t)[:10]  # YYYY‑MM‑DD
-        da_t = aligned_cube.sel(time=t)
-
-        # Ensure (band, y, x) memory layout for rasterio
-        data = da_t.transpose("band", "y", "x").values
-
-        profile = {
-            "driver": da_t.attrs.get("driver", "GTiff"),
-            "height": da_t.sizes["y"],
-            "width": da_t.sizes["x"],
-            "count": da_t.sizes["band"],
-            "dtype": str(da_t.dtype),
-            "transform": transform,
-            "crs": crs,
-            "nodata": int(getattr(da_t, "nodata", 0)),
-            "tiled": True,
-            "blockxsize": block_size,
-            "blockysize": block_size,
-            "interleave": "band",
-        }
-
-        outfile = out_dir_path / f"{date_str}.tif"
-        with rasterio.open(outfile, "w", **profile) as dst:
-            dst.write(data)

satcube-0.1.14/satcube/cloud_detection_old.py

@@ -1,24 +0,0 @@
-import torch
-
-class LandsatCloudDetector(torch.nn.Module):
-    def __init__(self):
-        super().__init__()
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        # Define bit flags for clouds based on the
-        # Landsat QA band documentation
-        cloud_flags = (1 << 3) | (1 << 4) | (1 << 1)
-
-        ## Get the QA band
-        qa_band = x[6]
-        mask_band = x[:6].mean(axis=0)
-        mask_band[~torch.isnan(mask_band)] = 1
-
-        ## Create a cloud mask
-        cloud_mask = torch.bitwise_and(qa_band.int(), cloud_flags) == 0
-        cloud_mask = cloud_mask.float()
-        cloud_mask[cloud_mask == 0] = torch.nan
-        cloud_mask[cloud_mask == 0] = 1
-        final_mask = cloud_mask * mask_band
-        return final_mask
-

satcube-0.1.14/satcube/dataclass.py

@@ -1,39 +0,0 @@
-import pathlib
-from datetime import datetime
-from typing import List, Optional
-
-import pydantic
-
-
-class Sensor(pydantic.BaseModel):
-    start_date: str
-    end_date: str
-    edge_size: int
-    bands: List[str]
-
-
-class Sentinel2(Sensor):
-    weight_path: pathlib.Path
-    start_date: Optional[str] = "2015-06-27"
-    end_date: Optional[str] = datetime.now().strftime("%Y-%m-%d")
-    resolution: Optional[int] = 10
-    edge_size: Optional[int] = 384
-    embedding_universal: Optional[str] = "s2_embedding_model_universal.pt"
-    cloud_model_universal: str = "s2_cloud_model_universal.pt"
-    cloud_model_specific: str = "s2_cloud_model_specific.pt"
-    super_model_specific: str = "s2_super_model_specific.pt"
-    bands: List[str] = [
-        "B01",
-        "B02",
-        "B03",
-        "B04",
-        "B05",
-        "B06",
-        "B07",
-        "B08",
-        "B8A",
-        "B09",
-        "B10",
-        "B11",
-        "B12",
-    ]