satcube 0.1.14__py3-none-any.whl → 0.1.15__py3-none-any.whl

satcube/__init__.py

@@ -1,10 +1,9 @@
 from satcube.cloud_detection import cloud_masking
 from satcube.download import download
 from satcube.align import align
-
-
-__all__ = ["cloud_masking", "download", "align"]
-
 import importlib.metadata
-__version__ = importlib.metadata.version("satcube")
+from satcube.objects  import SatCubeMetadata  
+
+__all__ = ["cloud_masking", "download", "align", "SatCubeMetadata"]
+# __version__ = importlib.metadata.version("satcube")
 

satcube/align.py

@@ -3,147 +3,96 @@ from __future__ import annotations
 import pathlib
 from typing import List, Tuple
 import pickle
+import pandas as pd
+import satalign
+import shutil
 
 import numpy as np
-import rasterio
+import rasterio as rio
 import xarray as xr
 from affine import Affine
-
-import satalign as sat
-
-
-def align(
-    input_dir: str | pathlib.Path,
-    output_dir: str | pathlib.Path,
-    *,
-    channel: str = "mean",
-    crop_center: int = 128,
-    num_threads: int = 2,
-    save_tiffs: bool = True,
-) -> Tuple[xr.DataArray, List[np.ndarray]]:
-    """Align all masked Sentinel‑2 tiles found in *input_dir*.
-
-    Parameters
-    ----------
-    input_dir
-        Directory containing masked Sentinel‑2 *TIFF* tiles produced by the
-        previous preprocessing stage.
-    output_dir
-        Directory where the alignment artefacts (``datacube.pickle``) and, if
-        requested, one aligned *GeoTIFF* per date will be written.
-    channel
-        Datacube band used by the *PCC* model for correlation.  ``"mean"`` is
-        recommended because it carries fewer noise artefacts.
-    crop_center
-        Half‑size (in pixels) of the square window extracted around the scene
-        centre that is fed to the correlation engine.
-    num_threads
-        Number of CPU threads for the multi‑core phase‑correlation run.
-    save_tiffs
-        If *True* (default) the aligned datacube is exported to tiled *COGs* via
-        :func:`save_aligned_cube_to_tiffs`.
-    pickle_datacube
-        If *True* (default) the raw (unaligned) datacube is pickled to
-        ``datacube.pickle`` inside *output_dir* for reproducibility/debugging.
-
-    Returns
-    -------
-    aligned_cube, warp_matrices
-        *aligned_cube* is the spatially aligned datacube as an
-        :class:`xarray.DataArray`; *warp_matrices* is the list of 3 × 3 affine
-        homography matrices (one per time step) returned by the
-        :pyclass:`~satalign.PCC` engine.
-    """
-    input_path = pathlib.Path(input_dir).expanduser().resolve()
-    output_path = pathlib.Path(output_dir).expanduser().resolve()
-    output_path.mkdir(parents=True, exist_ok=True)
-
-    # ── 1. Build datacube ────────────────────────────────────────────
-    da =  sat.utils.create_array(input_path)
-
-    # ── 2. Select reference slice (highest cloud‑score CDF) ───────────
-    da_sorted = da.sortby("cs_cdf", ascending=False)
-    ref_slice = da_sorted.isel(time=0)
-    reference = ref_slice.where((ref_slice != 0) & (ref_slice != 65535))
-
-    # ── 3. Instantiate and run PCC model ─────────────────────────────
-    pcc_model = sat.PCC(
-        datacube=da,
-        reference=reference,
-        channel=channel,
-        crop_center=crop_center,
-        num_threads=num_threads,
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+
+def process_row(row: pd.Series, reference: np.ndarray, input_dir: pathlib.Path, output_dir: pathlib.Path) -> None:
+    row_path = input_dir / (row["id"] + ".tif")
+    output_path = output_dir / (row["id"] + ".tif")
+    with rio.open(row_path) as src:
+        row_image = src.read()
+        profile_image = src.profile
+    
+    row_image_float = row_image.astype(np.float32) / 10000
+    row_image_float = row_image_float[np.newaxis, ...]
+    
+    pcc_model = satalign.LGM(
+        datacube     = row_image_float,
+        reference    = reference
     )
-    aligned_cube, warp_matrices = pcc_model.run_multicore()
-
-    # ── 4. Optionally export as Cloud‑Optimised GeoTIFFs ────────────
-    if save_tiffs:
-        save_aligned_cube_to_tiffs(aligned_cube, output_path)
-
-    return aligned_cube, warp_matrices
+    image, _ = pcc_model.run_multicore()
+    image = (image * 10000).astype(np.uint16).squeeze()
 
+    with rio.open(output_path, "w", **profile_image) as dst:
+        dst.write(image)
 
-def save_aligned_cube_to_tiffs(
-    aligned_cube: xr.DataArray,
-    out_dir: str | pathlib.Path,
-    *,
-    block_size: int = 128,
+def align(
+    input_dir: str | pathlib.Path = "raw",
+    output_dir: str | pathlib.Path = "aligned",
+    nworks: int = 4,
+    cache: bool = False
 ) -> None:
-    """Write each time slice of *aligned_cube* to an individual tiled COG.
-
-    The filenames follow the pattern ``YYYY‑MM‑DD.tif``.
-
-    Parameters
-    ----------
-    aligned_cube
-        Datacube returned by :func:`align_datacube`.
-    out_dir
-        Target directory; it will be created if it does not exist.
-    block_size
-        Internal tile size (*rasterio* ``blockxsize`` and ``blockysize``).
-    """
-    out_dir_path = pathlib.Path(out_dir).expanduser().resolve()
-    out_dir_path.mkdir(parents=True, exist_ok=True)
-
-    # ── 1. Build affine transform from x/y coordinate vectors ────────
-    x_vals = aligned_cube.x.values
-    y_vals = aligned_cube.y.values
-    x_res = float(x_vals[1] - x_vals[0])  # positive (east)
-    y_res = float(y_vals[1] - y_vals[0])  # negative (north‑up)
-    transform = (
-        Affine.translation(x_vals[0] - x_res / 2.0, y_vals[0] - y_res / 2.0)
-        * Affine.scale(x_res, y_res)
-    )
-
-    # ── 2. Retrieve CRS from datacube attributes ────────────────────
-    attrs = aligned_cube.attrs
-    crs: str | None = attrs.get("crs_wkt")
-    if crs is None and "crs_epsg" in attrs:
-        crs = f"EPSG:{int(attrs['crs_epsg'])}"
-
-    # ── 3. Loop over acquisition dates and write GeoTIFFs ────────────
-    for t in aligned_cube.time.values:
-        date_str = str(t)[:10]  # YYYY‑MM‑DD
-        da_t = aligned_cube.sel(time=t)
-
-        # Ensure (band, y, x) memory layout for rasterio
-        data = da_t.transpose("band", "y", "x").values
-
-        profile = {
-            "driver": da_t.attrs.get("driver", "GTiff"),
-            "height": da_t.sizes["y"],
-            "width": da_t.sizes["x"],
-            "count": da_t.sizes["band"],
-            "dtype": str(da_t.dtype),
-            "transform": transform,
-            "crs": crs,
-            "nodata": int(getattr(da_t, "nodata", 0)),
-            "tiled": True,
-            "blockxsize": block_size,
-            "blockysize": block_size,
-            "interleave": "band",
+    
+    input_dir = pathlib.Path(input_dir).expanduser().resolve()
+    output_dir = pathlib.Path(output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    metadata_path = input_dir / "metadata.csv"
+    
+    if not metadata_path.exists():
+        raise FileNotFoundError(
+            f"Metadata file not found: {metadata_path}. "
+            "Please run the download step first."
+        )
+    else:
+        metadata = pd.read_csv(metadata_path)
+        
+    if cache:
+        exist_files = [file.stem for file in output_dir.glob("*.tif")]
+        metadata = metadata[~metadata["id"].isin(exist_files)]
+        
+        if metadata.empty:
+            return
+
+    id_reference = metadata.sort_values(
+        by=["cs_cdf", "date"],
+        ascending=False,
+    ).iloc[0]["id"]
+    
+    reference_path = input_dir / (id_reference + ".tif")
+    
+    with rio.open(reference_path) as ref_src:
+        reference = ref_src.read()
+        
+    reference_float = reference.astype(np.float32) / 10000
+
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = {
+            executor.submit(process_row, row, reference_float, input_dir, output_dir)
+            for _, row in metadata.iterrows()
         }
-
-        outfile = out_dir_path / f"{date_str}.tif"
-        with rasterio.open(outfile, "w", **profile) as dst:
-            dst.write(data)
+        for future in tqdm(
+            as_completed(futures), 
+            total=len(futures), 
+            desc="Aligning images",
+            unit="image",
+            leave=True
+        ):
+            try:
+                future.result()
+            except Exception as e:
+                print(f"Error processing image: {e}")
+                
+    metadata = input_dir / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = output_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

satcube/cloud_detection.py

@@ -20,9 +20,11 @@ import rasterio as rio
 from rasterio.windows import Window
 import torch
 import pandas as pd
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from tqdm import tqdm
 import rasterio as rio
 from rasterio.merge import merge
+import shutil
 
 from satcube.utils import define_iteration, DeviceManager
 import warnings
@@ -35,7 +37,6 @@ warnings.filterwarnings(
 
 
 
-
 def infer_cloudmask(
     input_path: str | pathlib.Path,
     output_path: str | pathlib.Path,
@@ -68,13 +69,10 @@ def infer_cloudmask(
     -------
     pathlib.Path : The path to the created output image.
     """
-    # image_path = "/home/contreras/Documents/GitHub/satcube2/raw/2018-07-15_6q49m.tif"
-    # output_path = "/home/contreras/Documents/GitHub/satcube2/masked/2018-07-15_6q49m.tif"
-    
+
     input_path = pathlib.Path(input_path)
     output_path = pathlib.Path(output_path)
 
-    # 1) Validate metadata
     with rio.open(input_path) as src:
         meta = src.profile
         if not meta.get("tiled", False):
@@ -85,21 +83,13 @@ def infer_cloudmask(
                             f"got {meta['blockxsize']}x{meta['blockysize']}")
         height, width = meta["height"], meta["width"]
 
-
-    # 2) Crear un buffer en RAM (float32)
     full_mask = np.zeros((height, width), dtype=np.float32)
 
-    # 3) Iterar por ventanas
-
     coords = define_iteration((height, width), chunk_size, overlap)
 
     with rio.open(input_path) as src:
 
-        for (row_off, col_off) in tqdm(
-            coords, 
-            desc=f"{prefix} Inference on {input_path.name}",
-            position=1,
-            leave=False):
+        for (row_off, col_off) in coords: 
 
             window = Window(col_off, row_off, chunk_size, chunk_size)
             patch = src.read(window=window) / 1e4
@@ -143,9 +133,9 @@ def infer_cloudmask(
             with rio.open(output_mask, "w", **out_meta) as dst:
                 dst.write(full_mask, 1)
             
-        with rio.open(input_path) as src_img:
-            data = src_img.read()
-            img_prof = src_img.profile.copy()
+
+        data = src.read()
+        img_prof = src.profile.copy()
 
         masked = data.copy()
         masked[:, full_mask != 0] = 65535
@@ -160,9 +150,9 @@ def cloud_masking(
     input: str | pathlib.Path = "raw",
     output: str | pathlib.Path = "masked",
     model_path: str | pathlib.Path = "SEN2CloudEnsemble",
-    save_mask: bool = False,
     device: str = "cpu",
-    cache: bool = True
+    save_mask: bool = False,
+    nworks: int = 4,
 ) -> list[pathlib.Path]:
     """Write cloud-masked Sentinel-2 images.
 
@@ -214,29 +204,35 @@ def cloud_masking(
     model = mlstac.load(model_path)
     cloud_model = DeviceManager(model, init_device=device).model
     cloud_model.eval()
-
-    outs = [
-        infer_cloudmask(
-            input_path=p,
-            output_path=dst_dir / p.name,
-            cloud_model=cloud_model,
-            device=device,
-            save_mask=save_mask,
-            prefix=f"[{i+1}/{len(tif_paths)}] "
-        )
-        for i, p in enumerate(tif_paths)
-    ]
-
-    df_out = (
-        pd.Series(outs, name="path")
-        .to_frame()
-        .assign(path_str=lambda df: df["path"].astype(str))
-        .assign(
-            date  = lambda df: df["path_str"].str.extract(r"(\d{4}-\d{2}-\d{2})", expand=False),
-            score = lambda df: df["path_str"].str.extract(r"_(\d+)\.tif$", expand=False).astype(int) / 100
-        )
-        .drop(columns="path_str")                                    # ya no la necesitamos
-        .sort_values("score", ascending=False, ignore_index=True)
-    )
-
-    return df_out
+    
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = { 
+            executor.submit(
+                infer_cloudmask,
+                input_path=p,
+                output_path=dst_dir / p.name,
+                cloud_model=cloud_model,
+                device=device,
+                save_mask=save_mask,
+                prefix=f"[{i+1}/{len(tif_paths)}] "
+            ): p for i, p in enumerate(tif_paths)
+        }
+        
+        for future in tqdm(
+            as_completed(futures),
+            total=len(futures),
+            desc="Cloud Masking",
+            position=0,
+            leave=True
+        ):
+            p = futures[future]
+            try:
+                result = future.result()
+                print(f"{result} processed successfully.")
+            except Exception as e:
+                print(f"Error processing {p}: {e}")
+                
+    metadata = src / "metadata.csv"
+    if metadata.exists():
+        metadata_dst = dst_dir / "metadata.csv"
+        shutil.copy(metadata, metadata_dst)

satcube/download.py

@@ -1,6 +1,8 @@
 import sys, time, threading, itertools
 import cubexpress as ce
 import pandas as pd
+from satcube.objects import SatCubeMetadata
+import pathlib
 
 def download(
     lon: float,
@@ -12,54 +14,55 @@ def download(
     max_cscore: float = 1,
     min_cscore: float = 0,
     outfolder: str = "raw",
-    nworks: int = 4,
-    cache: bool = True,
-    show_spinner: bool = True,
-    verbose: bool = False
-) -> pd.DataFrame:
-    """
-    """
-    stop_flag = {"v": False}
+    nworks: int = 4
+) -> "SatCubeMetadata":
+        
+        
+    outfolder = pathlib.Path(outfolder).resolve()
 
-    if show_spinner and not verbose:
-        def _spin():
-            for ch in itertools.cycle("|/-\\"):
-                if stop_flag["v"]:
-                    break
-                sys.stdout.write(f"\rDownloading Sentinel-2 imagery & metadata… {ch}")
-                sys.stdout.flush()
-                time.sleep(0.1)
-            sys.stdout.write("\rDownloading Sentinel-2 imagery & metadata ✅\n")
-            sys.stdout.flush()
+    table = ce.s2_table(
+        lon=lon,
+        lat=lat,
+        edge_size=edge_size,
+        start=start,
+        end=end,
+        max_cscore=max_cscore,
+        min_cscore=min_cscore
+    )   
+        
+    requests = ce.table_to_requestset(
+        table=table,
+        mosaic=True
+    )
+    
+    ce.get_cube(
+        requests=requests,
+        outfolder=outfolder,
+        nworks=nworks
+    )
 
-        th = threading.Thread(target=_spin, daemon=True)
-        th.start()
-    else:
-        th = None
+    table_req = (
+        requests._dataframe.copy()
+        .drop(columns=['geotransform', 'manifest', 'outname', 'width', 'height', 'scale_x', 'scale_y'])
+    )
+    
+    table_req['date'] = table_req['id'].str.split('_').str[0]
 
-    try:
-        table = ce.s2_cloud_table(
-            lon=lon,
-            lat=lat,
-            edge_size=edge_size,
-            start=start,
-            end=end,
-            max_cscore=max_cscore,
-            min_cscore=min_cscore,
-            cache=cache,
-            verbose=verbose
+    result_table = (
+        table.groupby('date')
+        .agg(
+            id=('id', lambda x: '-'.join(x)),
+            cs_cdf=('cs_cdf', 'first')
         )
+        .reset_index()  
+    )
+    
+    table_final = table_req.merge(
+        result_table,
+        on='date',
+        how='left'
+    ).rename(columns={'id_x': 'id', 'id_y': 'gee_ids'})
 
-        ce.get_cube(
-            table=table,
-            outfolder=outfolder,
-            nworks=nworks,
-            verbose=verbose,
-            cache=cache
-        )
-    finally:
-        stop_flag["v"] = True
-        if th is not None:
-            th.join()
+    table_final.to_csv(outfolder / "metadata.csv", index=False)
 
-    return table
+    return SatCubeMetadata(df=table_final, raw_dir=outfolder)

satcube/objects.py

@@ -0,0 +1,71 @@
+# satcube/objects.py
+from __future__ import annotations
+from dataclasses import dataclass, field
+import pathlib
+import pandas as pd
+
+from satcube.align  import align as _align_fn
+from satcube.cloud_detection  import cloud_masking as _cloud_fn
+
+@dataclass
+class SatCubeMetadata:
+    df: pd.DataFrame = field(repr=False)
+    raw_dir: pathlib.Path = field(repr=False)
+
+    def __repr__(self) -> str: 
+        return self.df.__repr__()
+    
+    __str__ = __repr__
+    
+    def _repr_html_(self) -> str:
+        html = getattr(self.df, "_repr_html_", None)
+        return html() if callable(html) else self.df.__repr__()
+
+    def align(
+        self,
+        input_dir: str | pathlib.Path | None = None,
+        output_dir: str | pathlib.Path = "aligned",
+        nworks: int = 4,
+        cache: bool = False
+    ) -> "SatCubeMetadata":
+        
+        if input_dir is None:
+            input_dir = self.raw_dir
+        else:
+            input_dir = pathlib.Path(input_dir).expanduser().resolve()
+
+        _align_fn(
+            input_dir=input_dir, 
+            output_dir=output_dir, 
+            nworks=nworks, 
+            cache=cache
+        )
+        self.aligned_dir = pathlib.Path(output_dir).resolve()
+        return self
+
+    def cloud_masking(
+        self,
+        output_dir: str | pathlib.Path = "masked",
+        model_path: str | pathlib.Path = "SEN2CloudEnsemble",
+        device: str = "cpu"
+    ) -> "SatCubeMetadata":
+        if not hasattr(self, "aligned_dir"):
+            raise RuntimeError("You must run .align() first")
+        _cloud_fn(
+            input=self.aligned_dir,
+            output=output_dir,
+            model_path=model_path,
+            device=device
+        )
+        self.masked_dir = pathlib.Path(output_dir).resolve()
+        return self
+    
+    def __getattr__(self, item):
+        return getattr(self.df, item)
+
+    def __getitem__(self, key):
+        return self.df.__getitem__(key)
+    
+    def __len__(self):
+        return len(self.df)
+    

{satcube-0.1.14.dist-info → satcube-0.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: satcube
-Version: 0.1.14
+Version: 0.1.15
 Summary: A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data.
 Home-page: https://github.com/IPL-UV/satcube
 Author: Cesar Aybar
@@ -16,9 +16,10 @@ Requires-Dist: cubexpress (>=0.1.10)
 Requires-Dist: mlstac (>=0.4.0)
 Requires-Dist: phicloudmask (>=0.0.2)
 Requires-Dist: requests (>=2.26.0)
-Requires-Dist: satalign (>=0.1.12)
+Requires-Dist: satalign (>=0.1.9)
 Requires-Dist: scikit-learn (>=1.2.0)
 Requires-Dist: segmentation-models-pytorch (>=0.3.0)
+Requires-Dist: tqdm (>=4.67.1)
 Requires-Dist: xarray (>=2023.7.0)
 Project-URL: Documentation, https://ipl-uv.github.io/satcube/
 Project-URL: Repository, https://github.com/IPL-UV/satcube

satcube-0.1.15.dist-info/RECORD

@@ -0,0 +1,10 @@
+satcube/__init__.py,sha256=1q5c8Kx1dePuPBo29O7-1MA0L_GdBhi56gbxjQAa1_o,317
+satcube/align.py,sha256=0QOzX9f-r05F8niI91xlvifmyHaAQjWgBit_4Jc6OeM,2979
+satcube/cloud_detection.py,sha256=05OifUtWjE5jco8Q1jf_7qoQ45qrnJQgRc6LrPlMkLI,7898
+satcube/download.py,sha256=o8kwPzipU1f_pV0KNPLmLmjUEsoWb6yhkCt7kerDViU,1545
+satcube/objects.py,sha256=zhiYu-rUTm65TuEl2qIo7kLfpdam17_6CZl2KBCT9Jw,1988
+satcube/utils.py,sha256=QBdmSg6_4Vy-4mXH1Z3Z2AvbIKAXBYi5g3-IgWSE1MY,3660
+satcube-0.1.15.dist-info/LICENSE,sha256=YdB4BQMkMzWuKvXRIpQR4g91IQ_pwA5PSH2lNM97zFI,1070
+satcube-0.1.15.dist-info/METADATA,sha256=iyw3WijWmTfyg0-c-1qdxZ8UkgTd7XbnOnzZAs2QAV4,6558
+satcube-0.1.15.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+satcube-0.1.15.dist-info/RECORD,,

satcube/cloud_detection_old.py

@@ -1,24 +0,0 @@
-import torch
-
-class LandsatCloudDetector(torch.nn.Module):
-    def __init__(self):
-        super().__init__()
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        # Define bit flags for clouds based on the
-        # Landsat QA band documentation
-        cloud_flags = (1 << 3) | (1 << 4) | (1 << 1)
-
-        ## Get the QA band
-        qa_band = x[6]
-        mask_band = x[:6].mean(axis=0)
-        mask_band[~torch.isnan(mask_band)] = 1
-
-        ## Create a cloud mask
-        cloud_mask = torch.bitwise_and(qa_band.int(), cloud_flags) == 0
-        cloud_mask = cloud_mask.float()
-        cloud_mask[cloud_mask == 0] = torch.nan
-        cloud_mask[cloud_mask == 0] = 1
-        final_mask = cloud_mask * mask_band
-        return final_mask
-

satcube/dataclass.py

@@ -1,39 +0,0 @@
-import pathlib
-from datetime import datetime
-from typing import List, Optional
-
-import pydantic
-
-
-class Sensor(pydantic.BaseModel):
-    start_date: str
-    end_date: str
-    edge_size: int
-    bands: List[str]
-
-
-class Sentinel2(Sensor):
-    weight_path: pathlib.Path
-    start_date: Optional[str] = "2015-06-27"
-    end_date: Optional[str] = datetime.now().strftime("%Y-%m-%d")
-    resolution: Optional[int] = 10
-    edge_size: Optional[int] = 384
-    embedding_universal: Optional[str] = "s2_embedding_model_universal.pt"
-    cloud_model_universal: str = "s2_cloud_model_universal.pt"
-    cloud_model_specific: str = "s2_cloud_model_specific.pt"
-    super_model_specific: str = "s2_super_model_specific.pt"
-    bands: List[str] = [
-        "B01",
-        "B02",
-        "B03",
-        "B04",
-        "B05",
-        "B06",
-        "B07",
-        "B08",
-        "B8A",
-        "B09",
-        "B10",
-        "B11",
-        "B12",
-    ]