cars 1.0.0rc3__cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

cars/extractroi.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+cars-extractroi: helper to extract region of interest from image product
+"""
+
+import argparse
+import os
+
+import numpy as np
+import rasterio as rio
+from affine import Affine
+from shapely.geometry import box
+from shareloc.geomodels.rpc_writers import write_rio_rpc_as_rpb
+
+
+def is_bbx_in_image(bbx, image_dataset):
+    """
+    Checks if the bounding box is within the image.
+
+    Parameters:
+        bbx (array): The bounding box of the image.
+        image_dataset (rio.DatasetReader): Opened image dataset.
+
+    """
+    image_box = box(0, 0, image_dataset.height, image_dataset.width)
+
+    return image_box.contains(bbx)
+
+
+def get_slices_from_bbx(image_dataset, bbx, rpc_options):
+    """get slices from bounding box
+
+    Parameters:
+        image_dataset (rio.DatasetReader): Opened image dataset.
+        bbx (array): The bounding box of the image.
+        rpc_options (dict): Options for GDALCreateRPCTransformer.
+
+    Returns:
+        tuple: The slices from the bounding box.
+    """
+    transformer = rio.transform.RPCTransformer(
+        image_dataset.rpcs, **rpc_options
+    )
+    coordinates = [
+        transformer.rowcol(bbx[0], bbx[1]),
+        transformer.rowcol(bbx[2], bbx[1]),
+        transformer.rowcol(bbx[2], bbx[3]),
+        transformer.rowcol(bbx[0], bbx[3]),
+    ]
+    coordinates = np.array(coordinates)
+    (row_start, col_start) = np.amin(coordinates, axis=0)
+    (row_stop, col_stop) = np.amax(coordinates, axis=0)
+    rows = (row_start, row_stop)
+    cols = (col_start, col_stop)
+
+    return rows, cols
+
+
+def process_image_file(
+    bbx, input_image_path, output_image_path, rpb_file_path, rpc_options
+):
+    """
+    Processes an image file by extracting a region based on the given geometry.
+
+    Parameters:
+        region_geometry (dict): GeoJSON-like dictionary defining the region.
+        input_image_path (str): Path to the input image file.
+        output_image_path (str): Path to save the output image.
+        rpb_file_path (str): Path to save the .RPB file.
+        rpc_options (dict): Options for GDALCreateRPCTransformer.
+    """
+
+    with rio.open(input_image_path) as image_dataset:
+        if not image_dataset.rpcs:
+            raise ValueError("Image dataset has no RPCs")
+        validate_bounding_box(bbx, image_dataset, rpc_options)
+        row, col = get_slices_from_bbx(image_dataset, bbx, rpc_options)
+        window = rio.windows.Window.from_slices(row, col)
+        array = image_dataset.read(window=window)
+        profile = image_dataset.profile
+        profile["driver"] = "GTiff"
+        profile["width"] = window.width
+        profile["height"] = window.height
+        profile["transform"] = Affine.translation(
+            window.col_off, window.row_off
+        )
+        if "crs" in profile:
+            del profile["crs"]
+        with rio.open(output_image_path, "w", **profile) as dst:
+            # write data
+            dst.write(array)
+            # copy rpc
+            dst.rpcs = image_dataset.rpcs
+
+        if rpb_file_path is not None:
+            create_rpb_file(image_dataset, rpb_file_path)
+
+
+def get_human_readable_bbox(image_dataset, rpc_options):
+    """
+    Get the human-readable bounding box from an image dataset.
+
+    Parameters:
+        image_dataset (rio.DatasetReader): Opened image dataset.
+        rpc_options (dict): Options for GDALCreateRPCTransformer.
+
+    Returns:
+        tuple: The human-readable bounding box in the format
+        (min_x, max_x, min_y, max_y).
+    """
+
+    transformer = rio.transform.RPCTransformer(
+        image_dataset.rpcs, **rpc_options
+    )
+
+    human_readable_bbx = [
+        transformer.xy(0, 0),
+        transformer.xy(image_dataset.height, image_dataset.width),
+    ]
+    # fix coordinates to precision -7 for (x, y)
+    image_coords = [
+        (round(coord[0], 7), round(coord[1], 7)) for coord in human_readable_bbx
+    ]
+    [(x_1, y_1), (x_2, y_2)] = image_coords
+
+    min_x, max_x = min(x_1, x_2), max(x_1, x_2)
+    min_y, max_y = min(y_1, y_2), max(y_1, y_2)
+
+    return min_x, max_x, min_y, max_y
+
+
+def validate_bounding_box(bbx, image_dataset, rpc_options):
+    """
+    Validate the bounding box coordinates.
+
+    Parameters:
+        bbx (array): The bounding box of the image.
+        image_dataset (rio.DatasetReader): Opened image dataset.
+        rpc_options (dict): Options for GDALCreateRPCTransformer.
+    """
+
+    transformer = rio.transform.RPCTransformer(
+        image_dataset.rpcs, **rpc_options
+    )
+    input_box = box(
+        *transformer.rowcol(bbx[0], bbx[1]), *transformer.rowcol(bbx[2], bbx[3])
+    )
+    if not is_bbx_in_image(input_box, image_dataset):
+        min_x, max_x, min_y, max_y = get_human_readable_bbox(
+            image_dataset, rpc_options
+        )
+        raise ValueError(
+            f"Coordinates must be between "
+            f"({min_x}, {min_y}) and ({max_x}, {max_y})"
+        )
+
+
+def create_rpb_file(image_dataset, rpb_filename):
+    """
+    Create and save a .RPB file from a rasterio dataset
+
+    Parameters:
+        image_dataset (rio.DatasetReader): Opened image dataset.
+        rpb_filename (str): Path to save the .RPB file.
+    """
+    if not image_dataset.rpcs:
+        raise ValueError("Image dataset has no RPCs")
+    rpcs_as_dict = image_dataset.rpcs.to_dict()
+    write_rio_rpc_as_rpb(rpcs_as_dict, rpb_filename)
+
+
+def main():
+    """
+    Main cars-extractroi entrypoint
+    """
+    parser = argparse.ArgumentParser(
+        "cars-extractroi", description="Helper to extract roi from bounding box"
+    )
+    parser.add_argument(
+        "-il",
+        type=str,
+        nargs="*",
+        help="Image products",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-out",
+        type=str,
+        help="Extracts directory",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-bbx",
+        type=float,
+        nargs=4,
+        help="Bounding box from two points (x1, y1) and (x2, y2)",
+        metavar=("x1", "y1", "x2", "y2"),
+        required=True,
+    )
+
+    parser.add_argument(
+        "--rpc_height",
+        type=float,
+        help="Constant height offset used for projection",
+    )
+
+    parser.add_argument(
+        "--rpc_dem",
+        type=str,
+        help="Digital Elevation Model used for projection",
+    )
+
+    parser.add_argument(
+        "--generate_rpb",
+        action="store_true",
+        help="Generate RPB file",
+    )
+
+    args = parser.parse_args()
+    if not os.path.exists(args.out):
+        os.makedirs(args.out)
+
+    rpc_options = {}
+
+    if args.rpc_height is not None:
+        rpc_options["rpc_height"] = args.rpc_height
+    if args.rpc_dem is not None:
+        rpc_options["rpc_dem"] = args.rpc_dem
+
+    # check first input in list to determine pipeline
+    for idx, image_path in enumerate(args.il):
+        output_image_path = os.path.join(args.out, "ext_%03d.tif" % idx)
+        rpb_file_path = None
+        if args.generate_rpb:
+            rpb_file_path = os.path.splitext(output_image_path)[0] + ".RPB"
+
+        process_image_file(
+            args.bbx, image_path, output_image_path, rpb_file_path, rpc_options
+        )
+
+
+if __name__ == "__main__":
+    main()

cars/orchestrator/__init__.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Cars orchestrator module init file
+"""

cars/orchestrator/achievement_tracker.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+this module contains the achievement tracker
+"""
+
+import logging
+
+import numpy as np
+
+from cars.orchestrator.registry.abstract_registry import (
+    AbstractCarsDatasetRegistry,
+)
+
+
+class AchievementTracker:
+    """
+    AchievementTracker
+    """
+
+    def __init__(self):
+        """
+        Init function of AchievementTracker
+
+        """
+        self.tracked_cars_ds = []
+        self.cars_ds_ids = []
+        self.achievement = []
+
+    def track(self, cars_ds, cars_ds_id):
+        """
+        Track cars dataset
+
+        :param cars_ds: cars dataset to track
+        :type cars_ds: CarsDataset
+        :param cars_ds_id: id of cars dataset
+        :type cars_ds_id: int
+        """
+
+        if cars_ds not in self.tracked_cars_ds:
+            self.tracked_cars_ds.append(cars_ds)
+            self.cars_ds_ids.append(cars_ds_id)
+            self.achievement.append(np.zeros(cars_ds.shape, dtype=bool))
+
+    def add_tile(self, tile):
+        """
+        Add finished tile
+
+        :param tile: finished tile
+        :type tile: xarray Dataset or Pandas Dataframe
+        """
+
+        try:
+            self._add_tile(tile)
+        except RuntimeError:
+            logging.error("Error getting id in Achiement Tracker")
+
+    def _add_tile(self, tile):
+        """
+        Add finished tile
+
+        :param tile: finished tile
+        :type tile: xarray Dataset or Pandas Dataframe
+        """
+
+        # Get cars dataset id
+        cars_ds_id = AbstractCarsDatasetRegistry.get_future_cars_dataset_id(
+            tile
+        )
+        if cars_ds_id is None:
+            raise RuntimeError("No id in data")
+        if cars_ds_id not in self.cars_ds_ids:
+            raise RuntimeError("Cars ds not registered")
+        index = self.cars_ds_ids.index(cars_ds_id)
+
+        # Get position
+        row, col = AbstractCarsDatasetRegistry.get_future_cars_dataset_position(
+            tile
+        )
+        if None in (row, col):
+            logging.error("None in row, col in achievement tracker")
+        else:
+            # update
+            self.achievement[index][row, col] = 1
+
+    def get_remaining_tiles(self):
+        """
+        Get remaining tiles to compute
+
+        :return: remaining tiles
+        :rtype: list(delayed)
+        """
+
+        tiles = []
+
+        for cars_ds, achievement in zip(  # noqa: B905
+            self.tracked_cars_ds, self.achievement
+        ):
+            for row in range(cars_ds.shape[0]):
+                for col in range(cars_ds.shape[1]):
+                    if (
+                        not achievement[row, col]
+                        and cars_ds[row, col] is not None
+                    ):
+                        tiles.append(cars_ds[row, col])
+
+        return tiles

cars/orchestrator/cluster/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+CARS cluster module init file
+"""
+# flake8: noqa: F401
+
+# CARS imports
+from cars.orchestrator.cluster.abstract_cluster import AbstractCluster
+
+from . import (
+    abstract_dask_cluster,
+    dask_cluster_tools,
+    local_dask_cluster,
+    mp_cluster,
+    pbs_dask_cluster,
+    sequential_cluster,
+    slurm_dask_cluster,
+)

cars/orchestrator/cluster/abstract_cluster.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains abstract function for Abstract Cluster
+"""
+
+# Standard imports
+import logging
+import os
+from abc import ABCMeta, abstractmethod
+from typing import Dict
+
+from cars.orchestrator.cluster.log_wrapper import cars_profile
+
+
+class AbstractCluster(metaclass=ABCMeta):
+    """
+    AbstractCluster
+    """
+
+    # Available cluster modes to instanciate AbstractCluster subclasses.
+    available_modes: Dict = {}
+
+    # Define abstract attributes
+
+    # cluster mode output directory
+    out_dir: str
+
+    def __new__(  # pylint: disable=too-many-positional-arguments
+        cls,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
+    ):
+        """
+        Return the required cluster
+        :raises:
+         - KeyError when the required cluster is not registered
+
+        :param conf_cluster: configuration for cluster
+        :param out_dir: output directory for results
+        :param launch_worker: launcher of the new worker
+        :return: a cltser object
+        """
+
+        cluster_mode = "multiprocessing"
+        if conf_cluster is not None:
+            if "mode" not in conf_cluster:
+                logging.warning("Cluster mode not defined, default is used")
+            else:
+                cluster_mode = conf_cluster["mode"]
+
+        if cluster_mode not in cls.available_modes:
+            logging.error("No mode named {} registered".format(cluster_mode))
+            raise KeyError("No mode named {} registered".format(cluster_mode))
+
+        logging.info("The AbstractCluster {} will be used".format(cluster_mode))
+
+        return super(AbstractCluster, cls).__new__(
+            cls.available_modes[cluster_mode]
+        )
+
+    @classmethod
+    def register_subclass(cls, *short_names: str):
+        """
+        Allows to register the subclass with its short name
+        :param short_names: the subclasses to be registered
+        :type short_names: string
+        """
+
+        def decorator(subclass):
+            """
+            Registers the subclass in the available methods
+            :param subclass: the subclass to be registered
+            :type subclass: object
+            """
+            for short_name in short_names:
+                cls.available_modes[short_name] = subclass
+            return subclass
+
+        return decorator
+
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
+    ):  # pylint: disable=W0613
+        """
+        Init function of AbstractCluster
+
+        :param conf_cluster: configuration for cluster
+        :param data_to_propagate: data to propagate to new cluster if reset
+        :type data_to_propagate: dict
+
+        """
+        self.out_dir = out_dir
+
+        # data to propagate
+        self.data_to_propagate = data_to_propagate
+
+        self.worker_log_dir = os.path.join(log_dir, "workers_log")
+        if not os.path.exists(self.worker_log_dir):
+            os.makedirs(self.worker_log_dir)
+
+        self.log_level = logging.getLogger().getEffectiveLevel()
+        handlers = logging.getLogger().handlers
+        for hand in handlers:
+            if "stdout" == hand.get_name():
+                self.log_level = hand.level
+
+        # Check conf
+        self.checked_conf_cluster = self.check_conf(conf_cluster)
+
+    @abstractmethod
+    def get_delayed_type(self):
+        """
+        Get delayed type
+        """
+
+    @abstractmethod
+    def cleanup(self, **kwargs):
+        """
+        Cleanup cluster
+        """
+
+    @abstractmethod
+    def check_conf(self, conf):
+        """
+        Check configuration
+
+        :param conf: configuration to check
+        :type conf: dict
+
+        :return: overloaded configuration
+        :rtype: dict
+
+        """
+
+    def get_conf(self):
+        """
+        Get overriden configuration
+
+
+        :return: overriden configuration
+        """
+
+        return self.checked_conf_cluster
+
+    def create_task(self, func, nout=1):
+        """
+        Create task
+
+        :param func: function
+        :param nout: number of outputs
+        """
+
+        def create_task_builder(*argv, **kwargs):
+            """
+            Create task builder
+
+            :param argv: list of input arguments
+            :param kwargs: list of named input arguments
+            """
+            wrapped_cars_profile = cars_profile_wrapper
+            additionnal_kwargs = {
+                "func": func,
+                "func_name": func.__name__,
+            }
+
+            return self.create_task_wrapped(wrapped_cars_profile, nout=nout)(
+                *argv, **kwargs, **additionnal_kwargs
+            )
+
+        return create_task_builder
+
+    @abstractmethod
+    def create_task_wrapped(self, func, nout=1):
+        """
+        Create task
+
+        :param func: function
+        :param nout: number of outputs
+        """
+
+    @abstractmethod
+    def start_tasks(self, task_list):
+        """
+        Start all tasks
+
+        :param task_list: task list
+        """
+
+    @abstractmethod
+    def scatter(self, data, broadcast=True):
+        """
+        Distribute data through workers
+
+        :param data: task data
+        """
+
+    @abstractmethod
+    def future_iterator(self, future_list, timeout=None):
+        """
+        Iterator, iterating on computed futures
+
+        :param future_list: future_list list
+        :param timeout: time to wait for next job
+        """
+
+
+def cars_profile_wrapper(*argv, **kwargs):
+    """
+    Create a wrapper for cars_profile to be used in cluster tasks
+
+    :param argv: args of func
+    :param kwargs: kwargs of func
+
+    """
+
+    func = kwargs["func"]
+    func_name = kwargs["func_name"]
+    kwargs.pop("func")
+    kwargs.pop("func_name")
+
+    res = cars_profile(name=func_name)(func)(*argv, **kwargs)
+
+    return res