satcube 0.1.16__tar.gz → 0.1.18__tar.gz

{satcube-0.1.16 → satcube-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: satcube
-Version: 0.1.16
+Version: 0.1.18
 Summary: A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data.
 Home-page: https://github.com/IPL-UV/satcube
 Author: Cesar Aybar

{satcube-0.1.16 → satcube-0.1.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name    = "satcube"
-version = "0.1.16"
+version = "0.1.18"
 description = "A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data."
 authors = ["Cesar Aybar <fcesar.aybar@uv.es>"]
 repository     = "https://github.com/IPL-UV/satcube"

{satcube-0.1.16 → satcube-0.1.18}/satcube/__init__.py

@@ -1,9 +1,7 @@
-from satcube.cloud_detection import cloud_masking
 from satcube.download import download
-from satcube.align import align
-import importlib.metadata
 from satcube.objects  import SatCubeMetadata  
+import importlib.metadata
 
-__all__ = ["cloud_masking", "download", "align", "SatCubeMetadata"]
+__all__ = ["download", "SatCubeMetadata"]
 # __version__ = importlib.metadata.version("satcube")
 

satcube-0.1.18/satcube/align.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import pathlib
+from typing import Tuple
+import pandas as pd
+import satalign
+import numpy as np
+import rasterio as rio
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+
+def _process_image(
+    image: np.ndarray, 
+    reference: np.ndarray, 
+    profile: dict,
+    output_path: pathlib.Path,
+) -> Tuple[float, float]:
+    
+    image_float = image.astype(np.float32) / 10000
+    image_float = image_float[np.newaxis, ...]
+    
+    image, M = satalign.LGM(
+        datacube=image_float,
+        reference=reference
+    ).run_multicore()
+    
+    image = (image * 10000).astype(np.uint16).squeeze()
+
+    with rio.open(output_path, "w", **profile) as dst:
+        dst.write(image)
+    
+    return M[0][0, 2], M[0][1, 2]
+
+def _process_row(
+    row: pd.Series, 
+    reference: np.ndarray, 
+    input_dir: pathlib.Path, 
+    output_dir: pathlib.Path
+) -> Tuple[str, float, float]:
+    
+    row_path = input_dir / (row["id"] + ".tif")
+    output_path = output_dir / (row["id"] + ".tif")
+    
+    with rio.open(row_path) as src:
+        image = src.read()
+        profile = src.profile
+    
+    dx_px, dy_px = _process_image(
+        image=image,
+        reference=reference,
+        profile=profile,
+        output_path=output_path
+    )
+    
+    return row["id"], dx_px, dy_px 
+
+def align_fn(
+    metadata: pd.DataFrame | None = None,
+    input_dir: str | pathlib.Path = "raw",
+    output_dir: str | pathlib.Path = "aligned",
+    nworks: int = 4,
+    cache: bool = False
+) -> pd.DataFrame | None:
+    
+    input_dir = pathlib.Path(input_dir).expanduser().resolve()
+    output_dir = pathlib.Path(output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+        
+    if metadata is None:
+        raise FileNotFoundError(
+            f"Add metadata file to do alignment."
+            "Please run the download step first."
+        )
+        
+
+    id_reference = metadata.sort_values(
+        by=["cs_cdf"],
+        ascending=False,
+    ).iloc[0]["id"]
+    
+    df = metadata.copy()
+    
+    if cache:
+        exist_files = [file.stem for file in output_dir.glob("*.tif")]
+        df = df[~df["id"].isin(exist_files)]
+        if df.empty:
+            return metadata
+        
+    reference_path = input_dir / (id_reference + ".tif")
+    
+    with rio.open(reference_path) as ref_src:
+        reference = ref_src.read()
+        
+    reference_float = reference.astype(np.float32) / 10000
+
+    results = []
+    
+    with ThreadPoolExecutor(max_workers=nworks) as executor:
+        futures = {
+            executor.submit(
+                _process_row, 
+                row=row, 
+                reference=reference_float, 
+                input_dir=input_dir, 
+                output_dir=output_dir
+            ): row["id"]
+            for _, row in df.iterrows()
+        }
+        
+        for future in tqdm(
+            as_completed(futures), 
+            total=len(futures), 
+            desc="Aligning images",
+            unit="image",
+            leave=True
+        ):
+            try:
+                img_id, dx_px, dy_px = future.result()
+                results.append({"id": img_id,
+                                "dx_px": dx_px,
+                                "dy_px": dy_px})
+            except Exception as e:
+                print(f"Error processing image: {e} {futures[future]}")
+    
+    shift_df = pd.DataFrame(results)
+    
+    metadata = metadata.drop(
+        columns=["dx_px","dy_px"], 
+        errors="ignore"
+    )
+    
+    metadata = metadata.merge(
+        shift_df,
+        on="id",
+        how="left",
+        suffixes=('', '') 
+    )
+    
+    return metadata

satcube-0.1.18/satcube/archive_cloud_detection.py

@@ -0,0 +1,23 @@
+import torch
+
+class LandsatCloudDetector(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # Define bit flags for clouds based on the
+        # Landsat QA band documentation
+        cloud_flags = (1 << 3) | (1 << 4) | (1 << 1)
+
+        ## Get the QA band
+        qa_band = x[6]
+        mask_band = x[:6].mean(axis=0)
+        mask_band[~torch.isnan(mask_band)] = 1
+
+        ## Create a cloud mask
+        cloud_mask = torch.bitwise_and(qa_band.int(), cloud_flags) == 0
+        cloud_mask = cloud_mask.float()
+        cloud_mask[cloud_mask == 0] = torch.nan
+        cloud_mask[cloud_mask == 0] = 1
+        final_mask = cloud_mask * mask_band
+        return final_mask

satcube-0.1.18/satcube/archive_dataclass.py

@@ -0,0 +1,39 @@
+import pathlib
+from datetime import datetime
+from typing import List, Optional
+
+import pydantic
+
+
+class Sensor(pydantic.BaseModel):
+    start_date: str
+    end_date: str
+    edge_size: int
+    bands: List[str]
+
+
+class Sentinel2(Sensor):
+    weight_path: pathlib.Path
+    start_date: Optional[str] = "2015-06-27"
+    end_date: Optional[str] = datetime.now().strftime("%Y-%m-%d")
+    resolution: Optional[int] = 10
+    edge_size: Optional[int] = 384
+    embedding_universal: Optional[str] = "s2_embedding_model_universal.pt"
+    cloud_model_universal: str = "s2_cloud_model_universal.pt"
+    cloud_model_specific: str = "s2_cloud_model_specific.pt"
+    super_model_specific: str = "s2_super_model_specific.pt"
+    bands: List[str] = [
+        "B01",
+        "B02",
+        "B03",
+        "B04",
+        "B05",
+        "B06",
+        "B07",
+        "B08",
+        "B8A",
+        "B09",
+        "B10",
+        "B11",
+        "B12",
+    ]

satcube-0.1.18/satcube/archive_main.py

@@ -0,0 +1,453 @@
+import pathlib
+from typing import Optional, Tuple, Union
+from datetime import datetime
+
+import fastcubo
+import pandas as pd
+import torch
+
+from satcube.archive_dataclass import Sensor
+from satcube.archive_utils import (aligned_s2, cloudmasking_s2, display_images,
+                           gapfilling_s2, intermediate_process, interpolate_s2,
+                           metadata_s2, monthly_composites_s2, smooth_s2, super_s2)
+
+
+class SatCube:
+    """Satellite cube class to create datacubes from a specific sensor."""
+
+    def __init__(
+        self,
+        sensor: Sensor,
+        output_dir: str,
+        max_workers: int,
+        coordinates: Tuple[float, float],
+        device: Union[str, torch.device],
+    ):
+        """Create a new instance of the Satellite Cube class.
+
+        Args:
+            coordinates (Tuple[float, float]): The coordinates of the
+                location to download the data.
+            sensor (Sensor): The sensor object with all the information
+                to download and preprocess the data.
+            output_dir (str): The output directory to save the data.
+            max_workers (int): The maximum number of workers to use in the
+                download process.
+            device (Union[str, torch.device]): The device to use in the
+                cloud removal process.
+        """
+        self.device = device
+        self.sensor = sensor
+        self.output_dir = pathlib.Path(output_dir)
+        self.max_workers = max_workers
+        self.lon, self.lat = coordinates
+
+        # If the output directory does not exist, create it
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+    def metadata_s2(
+        self,
+        out_csv: Optional[pathlib.Path] = None,
+        quiet: Optional[pathlib.Path] = False,
+        force: Optional[bool] = False,
+    ) -> pd.DataFrame:
+        """Create a pd.DataFrame with the images to download and their
+        cloud cover percentage. The table is saved in a CSV file.
+
+        Args:
+            force (Optional[bool], optional): If True, the query
+                process is done again. Defaults to False.
+            out_csv (Optional[pathlib.Path], optional): The path to the
+                CSV file with the query table. Defaults to None.
+            quiet (Optional[bool], optional): If True, no message is
+                displayed. Defaults to False.
+            force (Optional[bool], optional): If True, the query process
+                is done again. Defaults to False.
+
+        Returns:
+            pd.DataFrame: The table with the images to download.
+        """
+
+        if out_csv is None:
+            out_csv: pathlib.Path = self.output_dir / "s2_01_gee_query.csv"
+        
+        if not out_csv.exists() or force:
+            query_table: pd.DataFrame = metadata_s2(
+                lon=self.lon,
+                lat=self.lat,
+                range_date=(self.sensor.start_date, self.sensor.end_date),
+                edge_size=self.sensor.edge_size,
+                quiet=quiet
+            )
+            query_table.to_csv(out_csv, index=False)
+        else:
+            query_table = pd.read_csv(out_csv)
+
+        return query_table
+
+    def download_s2_image(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Download the images from the query table.
+
+        Args:
+            table (pd.DataFrame): The table with the images to download.
+            out_csv (Optional[pathlib.Path], optional): The path to the
+                CSV file with the query table. Defaults to None.
+            quiet (Optional[bool], optional): If True, the download
+                process is not displayed. Defaults to False.
+            force (Optional[bool], optional): If True, the download
+                process is done again. Defaults to False.
+
+        Returns:
+            pathlib.Path: The path to the folder with the downloaded images.
+        """
+
+        # Create the output directory if it does not exist
+        if out_folder is None:
+            output_path: pathlib.Path = self.output_dir / "s2_01_raw"
+
+        # Download the selected images
+        if not output_path.exists() or force:
+            if not quiet:
+                print(f"Saving the images in the directory {output_path}")
+
+            fastcubo.getPixels(
+                table=table, nworkers=self.max_workers, output_path=output_path
+            )
+
+        # Add folder path
+        table["folder"] = output_path
+
+        return table
+
+    def cloudmasking_s2(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Remove the clouds from the data.
+
+        Args:
+            table (pd.DataFrame): The table with the images to remove
+                the clouds.
+            out_csv (Optional[pathlib.Path], optional): The path to the
+                CSV file with the query table. Defaults to None.
+            quiet (Optional[bool], optional): If True, the messages
+                are not displayed. Defaults to False.
+            force (Optional[bool], optional): If True, the cloud removal
+                is done again. Defaults to False.
+
+        Returns:
+            pathlib.Path: The path to the folder with the
+                data without clouds.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_02_nocloud"
+
+        # Apply the cloud removal
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=cloudmasking_s2,
+            process_function_args=dict(
+                device=self.device,
+                sensor=self.sensor,
+                quiet=quiet
+            ),
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        # Sort by cloud cover
+        out_table = out_table.sort_values(by="cloud_cover", ascending=False)
+        out_table.reset_index(drop=True, inplace=True)
+
+        return out_table
+
+    def gapfilling_s2(
+        self,
+        table: pd.DataFrame,
+        method: Optional[str] = "linear",
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Fill the gaps in the data.
+
+        Args:
+            force (Optional[bool], optional): If True, the gap filling
+                is done again. Defaults to False.
+            histogram_match_error (float, optional): If the error in the
+                histogram matching is greater than this value, the image
+                is not filled and therefore, it is removed. Defaults
+                to 0.10.
+
+        Returns:
+            pathlib.Path: The path to the folder with the
+                data without gaps.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_03_nogaps"
+
+        # Apply the cloud removal
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=gapfilling_s2,
+            process_function_args={"method": method, "quiet": quiet},
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        # Sort by the matching error
+        out_table = out_table.sort_values(by="match_error", ascending=False)
+
+        return out_table
+
+    def align_s2(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Align all the images in the data cube.
+
+        Args:
+            table (pd.DataFrame): The table with the images to align.
+            force (Optional[bool], optional): If True, the alignment
+                is done again. Defaults to False.
+
+        Returns:
+            pathlib.Path: The path to the folder with the
+                aligned images.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_04_aligned"
+
+        # Apply the cloud removal
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=aligned_s2,
+            process_function_args={"quiet": quiet},
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        return out_table
+    
+    def monthly_composites_s2(
+        self,
+        table: Optional[pd.DataFrame],
+        out_folder: Optional[pathlib.Path] = None,
+        agg_method: Optional[str] = "median",
+        date_range: Tuple[str, str] = ("2016-01-01", datetime.now().strftime("%Y-%m-%d")),
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Smooth the data considering the temporal dimension.
+
+        Args:
+            force (Optional[bool], optional): If True, the interpolation
+                is done again. Defaults to False.
+
+        Returns:
+            xr.Dataset: The interpolated data.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_05_monthlycomposites"
+
+        # Prepare the composites
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=monthly_composites_s2,
+            process_function_args=dict(
+                agg_method=agg_method,
+                date_range=date_range,
+                quiet=quiet
+            ),
+            force=force
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        return out_table
+
+    def interpolate_s2(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pathlib.Path:
+        """Interpolate the data.
+
+        Args:
+            force (Optional[bool], optional): If True, the interpolation
+                is done again. Defaults to False.
+
+        Returns:
+            xr.Dataset: The interpolated data.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_06_interpolation"
+        
+        # Apply the cloud removal
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=interpolate_s2,
+            process_function_args=dict(
+                quiet=quiet
+            ),
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        return out_table
+    
+    def smooth_s2(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        smooth_w: Optional[int] = 7,
+        smooth_p: Optional[int] = 1,
+        device: Union[str, torch.device, None] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pd.DataFrame:
+        """Interpolate the data.
+
+        Args:
+            force (Optional[bool], optional): If True, the interpolation
+                is done again. Defaults to False.
+
+        Returns:
+            xr.Dataset: The interpolated data.
+        """        
+        if out_folder is None:
+            out_folder: pathlib = self.output_dir / "s2_07_smoothed"
+
+        # Apply the cloud removal
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=smooth_s2,
+            process_function_args=dict(
+                quiet=quiet,
+                smooth_w=smooth_w,
+                smooth_p=smooth_p,
+                device=device if device is not None else self.device
+            ),
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        return out_table
+
+
+    def super_s2(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        quiet: Optional[bool] = False,
+        force: Optional[bool] = False,
+    ) -> pd.DataFrame:
+        """Superresolution to the Sentinel-2 image cube.
+
+        Args:
+            table (pd.DataFrame): The table with the images to
+                superresolve.
+            out_folder (Optional[pathlib.Path], optional): The path to the
+                CSV file with the query table. Defaults to None.
+            quiet (Optional[bool], optional): If True, the messages
+                are not displayed. Defaults to False.
+            force (Optional[bool], optional): If True, the superresolution
+                is done again. Defaults to False.
+
+        Returns:
+            pd.DataFrame: The table with the superresolved images.
+        """
+
+        if out_folder is None:
+            out_folder: pathlib.Path = self.output_dir / "s2_08_superresolution"
+
+        # Apply the Superresolution process
+        out_table = intermediate_process(
+            table=table,
+            out_folder=out_folder,
+            process_function=super_s2,
+            process_function_args=dict(
+                device=self.device,
+                sensor=self.sensor,
+                quiet=quiet
+            ),
+            force=force,
+        )
+
+        # Change the folder path
+        out_table["folder"] = out_folder
+
+        return out_table
+
+    def display_images(
+        self,
+        table: pd.DataFrame,
+        out_folder: Optional[pathlib.Path] = None,
+        bands: Optional[list[str]] = [2, 1, 0],
+        ratio: Optional[int] = 3000,
+    ) -> pathlib.Path:
+        """ Display the images in the table.
+
+        Args:
+            table (pd.DataFrame): The table with the images to display.
+            out_folder (Optional[pathlib.Path], optional): The path to the
+                CSV file with the query table. Defaults to None.
+            bands (Optional[list[str]], optional): The bands to display.
+                Defaults to [2, 1, 0].
+            ratio (Optional[int], optional): The ratio to divide the
+                image. Defaults to 3000. The larger the number, the
+                darker the image.
+
+        Returns:
+            pathlib.Path: The path to the folder with the
+                displayed images.
+        """
+
+        # Create the output folder
+        out_folder = (
+            self.output_dir / ("z_" + table["folder"].iloc[0].name + "_png")
+        )
+        out_folder.mkdir(exist_ok=True, parents=True)
+        
+        return display_images(
+            table=table,
+            out_folder=out_folder,
+            bands=bands,
+            ratio=ratio,
+        )