cars 1.0.0a2__cp310-cp310-win_amd64.whl → 1.0.0a4__cp310-cp310-win_amd64.whl

cars/core/projection.py

@@ -22,7 +22,7 @@
 Projection module:
 contains some general purpose functions using polygons and data projections
 """
-# pylint: disable=too-many-lines
+# pylint: disable=C0302(too-many-lines)
 
 # Standard imports
 import logging
@@ -42,6 +42,7 @@ from shapely.ops import transform
 
 from cars.core import constants as cst
 from cars.core import inputs, outputs, utils
+from cars.orchestrator.cluster.log_wrapper import cars_profile
 
 
 def compute_dem_intersection_with_poly(  # noqa: C901
@@ -182,6 +183,22 @@ def polygon_projection(poly: Polygon, epsg_in: int, epsg_out: int) -> Polygon:
     return poly
 
 
+def polygon_projection_crs(poly: Polygon, crs_in: CRS, crs_out: CRS) -> Polygon:
+    """
+    Projects a polygon from an initial crs to another
+
+    :param poly: poly to project
+    :param crs_in: initial crs
+    :param crs_out: final crs
+    :return: The polygon in the final projection
+    """
+    # Project polygon between CRS (keep always_xy for compatibility)
+    project = pyproj.Transformer.from_crs(crs_in, crs_out, always_xy=True)
+    poly = transform(project.transform, poly)
+
+    return poly
+
+
 def geo_to_ecef(
     lat: np.ndarray, lon: np.ndarray, alt: np.ndarray
 ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
@@ -203,7 +220,7 @@ def geo_to_ecef(
     )[0]
 
 
-def ecef_to_enu(
+def ecef_to_enu(  # pylint: disable=too-many-positional-arguments
     x_ecef: np.ndarray,
     y_ecef: np.ndarray,
     z_ecef: np.ndarray,
@@ -253,7 +270,7 @@ def ecef_to_enu(
     return x_east, y_north, z_up
 
 
-def geo_to_enu(
+def geo_to_enu(  # pylint: disable=too-many-positional-arguments
     lat: np.ndarray,
     lon: np.ndarray,
     alt: np.ndarray,
@@ -302,7 +319,7 @@ def enu_to_aer(
     return azimuth, elevation, xyz_range
 
 
-def geo_to_aer(
+def geo_to_aer(  # pylint: disable=too-many-positional-arguments
     lat: np.ndarray,
     lon: np.ndarray,
     alt: np.ndarray,
@@ -351,6 +368,27 @@ def point_cloud_conversion(
     return cloud_in
 
 
+def point_cloud_conversion_crs(
+    cloud_in: np.ndarray, crs_in: int, crs_out: int
+) -> np.ndarray:
+    """
+    Convert a point cloud from a SRS to another one.
+
+    :param cloud_in: cloud to project
+    :param crs_in: crs of the input SRS
+    :param crs_out: crs of the output SRS
+    :return: Projected point cloud
+    """
+    # Project point cloud between CRS (keep always_xy for compatibility)
+    cloud_in = np.array(cloud_in).T
+    transformer = pyproj.Transformer.from_crs(crs_in, crs_out, always_xy=True)
+
+    cloud_in = transformer.transform(*cloud_in)
+    cloud_in = np.array(cloud_in).T
+
+    return cloud_in
+
+
 def get_xyz_np_array_from_dataset(
     cloud_in: xr.Dataset,
 ) -> Tuple[np.array, List[int]]:
@@ -492,6 +530,8 @@ def ground_polygon_from_envelopes(
     return inter, inter.bounds
 
 
+# pylint: disable=too-many-positional-arguments
+@cars_profile(name="Ground intersection envelopes", interval=0.5)
 def ground_intersection_envelopes(
     sensor1,
     sensor2,
@@ -558,7 +598,7 @@ def ground_intersection_envelopes(
     return inter_poly, (inter_xmin, inter_ymin, inter_xmax, inter_ymax)
 
 
-def get_ground_direction(
+def get_ground_direction(  # pylint: disable=too-many-positional-arguments
     sensor,
     geomodel,
     geometry_plugin: str,
@@ -631,7 +671,7 @@ def get_ground_direction(
     return np.array([lat0, lon0, alt0, lat, lon, alt])
 
 
-def get_ground_angles(
+def get_ground_angles(  # pylint: disable=too-many-positional-arguments
     sensor1,
     sensor2,
     geomodel1,
@@ -721,3 +761,83 @@ def get_ground_angles(
     convergence_angle = np.degrees(utils.angle_vectors(enu1, enu2))
 
     return az1, elev_angle1, az2, elev_angle2, convergence_angle
+
+
+def get_output_crs(epsg, out_conf):
+    """
+    Détermine le CRS de sortie en fonction de la config.
+    """
+    geoid = out_conf.get("geoid")
+    crs_epsg = CRS(f"EPSG:{epsg}")
+
+    if len(crs_epsg.axis_info) != 2:
+        return crs_epsg  # the user himself set a 3D CRS
+
+    geoid_is_path = isinstance(geoid, str)
+
+    if geoid_is_path:  # user given geoid
+        vepsg = guess_vcrs_from_file_name(geoid)
+        if vepsg is None:
+            custom_wkt = (
+                'VERTCRS["Custom geoid height",'
+                + f'    VDATUM["Custom geoid model (file: {geoid})"],'
+                + "    CS[vertical,1],"
+                + '    AXIS["gravity-related height (h)", up],'
+                + '    LENGTHUNIT["metre", 1, ID["EPSG", 9001]]'
+                "]"
+            )
+            logging.warning(
+                "Could not create a known VCRS from the geoid file."
+            )
+            return CRS.from_wkt(
+                f'COMPOUNDCRS["EPSG:{epsg} + Custom geoid height",'
+                f"    {crs_epsg.to_wkt()},"
+                f"    {custom_wkt}]"
+            )
+        # a vepsg was found using the geoid file
+        return CRS(f"EPSG:{epsg}+{vepsg}")
+
+    if geoid:  # geoid == True
+        return CRS(f"EPSG:{epsg}+5773")
+
+    # geoid == False
+    wgs84_wkt = (
+        'VERTCRS["WGS 84 ellipsoidal height",'
+        + '    VDATUM["WGS 84"],'
+        + "    CS[vertical,1],"
+        + '    AXIS["ellipsoidal height (h)", up],'
+        + '    LENGTHUNIT["metre", 1, ID["EPSG", 9001]]'
+        "]"
+    )
+    logging.warning("The output VCRS is WGS84.")
+
+    return CRS.from_wkt(
+        f'COMPOUNDCRS["EPSG:{epsg} + WGS84 ellipsoidal height",'
+        f"    {crs_epsg.to_wkt()},"
+        f"    {wgs84_wkt}]"
+    )
+
+
+def guess_vcrs_from_file_name(filepath):
+    """
+    Tries to detect the geoid's EPSG from the file name
+    """
+    filename = os.path.basename(filepath).lower()
+
+    known_models = {
+        "egm96": 5773,  # EGM96 height
+        "egm_96": 5773,  # alias
+        "egm 96": 5773,  # alias
+        "egm1996": 5773,  # alias
+        "egm08": 3855,  # EGM2008 height
+        "egm_08": 3855,  # alias
+        "egm 08": 3855,  # alias
+        "egm2008": 3855,  # alias
+    }
+
+    for key, vepsg in known_models.items():
+        if key in filename:
+            return vepsg
+
+    # aucun match connu
+    return None

cars/core/tiling.py

@@ -40,8 +40,10 @@ from scipy.spatial import tsearch  # pylint: disable=no-name-in-module
 from shapely.geometry import box, mapping
 from shapely.geometry.multipolygon import MultiPolygon
 
+from cars.orchestrator.cluster.log_wrapper import cars_profile
 
-def grid(
+
+def grid(  # pylint: disable=too-many-positional-arguments
     xmin: float, ymin: float, xmax: float, ymax: float, xsplit: int, ysplit: int
 ) -> np.ndarray:
     """
@@ -74,6 +76,7 @@ def grid(
     return out_grid
 
 
+@cars_profile(name="Transform four layers to two layers grid", interval=0.5)
 def transform_four_layers_to_two_layers_grid(tiling_grid, terrain=False):
     """
     Transform a 4 layer grid: (N, M, 4) containing
@@ -115,6 +118,7 @@ def transform_four_layers_to_two_layers_grid(tiling_grid, terrain=False):
     return arr
 
 
+@cars_profile(name="Transform disp range grid to two layers", interval=0.5)
 def transform_disp_range_grid_to_two_layers(disp_min_grid, disp_max_grid):
     """
     Transform tiling disp min and max to N+1 M+1 array corresponding
@@ -154,7 +158,7 @@ def transform_disp_range_grid_to_two_layers(disp_min_grid, disp_max_grid):
     return disp_min, disp_max
 
 
-def generate_tiling_grid(
+def generate_tiling_grid(  # pylint: disable=too-many-positional-arguments
     row_min: float,
     col_min: float,
     row_max: float,
@@ -199,7 +203,9 @@ def generate_tiling_grid(
     return out_grid
 
 
-def split(xmin, ymin, xmax, ymax, xsplit, ysplit):
+def split(
+    xmin, ymin, xmax, ymax, xsplit, ysplit
+):  # pylint: disable=too-many-positional-arguments
     """
     Split a region defined by [xmin, xmax] x [ymin, ymax]
         in splits of xsplit x ysplit size
@@ -561,6 +567,7 @@ def region_hash_string(region: Tuple):
     return "{}_{}_{}_{}".format(region[0], region[1], region[2], region[3])
 
 
+# pylint: disable=too-many-positional-arguments
 def get_corresponding_tiles_row_col(
     terrain_tiling_grid: np.ndarray,
     row: int,

cars/data_structures/cars_dataset.py

@@ -38,6 +38,7 @@ from typing import Dict
 # Third party imports
 import numpy as np
 import pandas
+import pyproj
 import rasterio as rio
 import xarray as xr
 from rasterio.profiles import DefaultGTiffProfile
@@ -338,7 +339,7 @@ class CarsDataset:
 
         return new_window
 
-    def create_grid(
+    def create_grid(  # pylint: disable=too-many-positional-arguments
         self,
         nb_col: int,
         nb_row: int,
@@ -434,7 +435,7 @@ class CarsDataset:
                 tiles_row.append(None)
             self.tiles.append(tiles_row)
 
-    def generate_descriptor(
+    def generate_descriptor(  # pylint: disable=too-many-positional-arguments
         self, future_result, file_name, tag=None, dtype=None, nodata=None
     ):
         """
@@ -578,7 +579,7 @@ def run_save_arrays(future_result, file_name, tag=None, descriptor=None):
     )
 
 
-def run_save_points(
+def run_save_points(  # pylint: disable=too-many-positional-arguments
     future_result,
     file_name,
     overwrite=False,
@@ -839,7 +840,7 @@ def save_single_tile_dict(dict_cars, tile_path_name: str):
     dict_cars.attrs = saved_dict_cars_attrs
 
 
-def fill_dataset(
+def fill_dataset(  # pylint: disable=too-many-positional-arguments
     dataset,
     saving_info=None,
     window=None,
@@ -938,7 +939,7 @@ def fill_dict(data_dict, saving_info=None, attributes=None):
             data_dict.attrs[SAVING_INFO] = saving_info
 
 
-def save_all_dataframe(
+def save_all_dataframe(  # pylint: disable=too-many-positional-arguments
     dataframe,
     file_name,
     save_by_pair=False,
@@ -1327,12 +1328,13 @@ def dict_profile_to_rio_profile(dict_profile: Dict) -> Dict:
     crs = None
     if "crs" in dict_profile:
         if dict_profile["crs"] is not None:
-            if isinstance(dict_profile["crs"], str):
-                crs = rio.crs.CRS.from_epsg(
-                    dict_profile["crs"].replace("EPSG:", "")
-                )
+            if (
+                isinstance(dict_profile["crs"], str)
+                and "EPSG:" in dict_profile["crs"]
+            ):
+                crs = pyproj.CRS(dict_profile["crs"].replace("EPSG:", ""))
             else:
-                crs = rio.crs.CRS.from_epsg(dict_profile["crs"])
+                crs = pyproj.CRS(dict_profile["crs"])
 
     rio_profile["crs"] = crs
     rio_profile["transform"] = transform

cars/data_structures/corresponding_tiles_tools.py

@@ -27,8 +27,6 @@ Contains functions for array reconstructions and crop for multiple tiles
 import numpy as np
 import xarray as xr
 
-import cars.core.constants as cst
-
 
 def reconstruct_data(tiles, window, overlap):  # noqa: C901
     """
@@ -186,104 +184,3 @@ def reconstruct_data(tiles, window, overlap):  # noqa: C901
             dims=dims,
         ).astype(data_type)
     return new_dataset, row_min - ol_row_min, col_min - ol_col_min
-
-
-def find_tile_dataset(corresponding_tiles, window):
-    """
-    Find the dataset corresponding to window, in the list of tiles.
-
-    :param corresponding_tiles: list of tiles
-    :type corresponding_tiles: list(tuple)
-    :param window: window of base tile [row min, row max, col min col max]
-    :type window: list
-
-    :return: dataset corresponding to window
-    :rtype: xr.Dataset
-
-    """
-
-    dataset = None
-    for tile_window, _, tile_dataset in corresponding_tiles:
-        if tuple(tile_window) == tuple(window):
-            dataset = tile_dataset
-
-    return dataset
-
-
-def crop_dataset(full_dataset, in_dataset, window, overlap, row_min, col_min):
-    """
-    Crop full dataset to fit with a given tile dataset
-
-    :param full_dataset: Combined dataset
-    :type full_dataset: xr.Dataset
-    :param in_dataset: dataset to use as template dataset
-    :type in_dataset: xr.Dataset
-    :param window: window of base tile [row min, row max, col min col max]
-    :type window: list
-    :param overlap: overlap of base tile [row min, row max, col min col max]
-    :type overlap: list
-    :param row_min: position of row min in full image
-    :type row_min: int
-    :param col_min: position of col min in full image
-    :type col_min: int
-
-    :return: cropped dataset
-    :rtype: xr.Dataset
-
-    """
-    cropped = xr.Dataset()
-
-    coords = {}
-    if cst.ROW in in_dataset:
-        coords[cst.ROW] = in_dataset.coords[cst.ROW]
-    if cst.COL in in_dataset:
-        coords[cst.COL] = in_dataset.coords[cst.COL]
-
-    list_tags = list(full_dataset.keys())
-
-    for tag in list_tags:
-        full_data = full_dataset[tag].values
-
-        offset_row = int(window[0] - overlap[0] - row_min)
-        offset_col = int(window[2] - overlap[2] - col_min)
-
-        if len(full_data.shape) == 2:
-            nb_row = int(in_dataset[tag].values.shape[0])
-            nb_col = int(in_dataset[tag].values.shape[1])
-
-            values = np.ascontiguousarray(
-                full_data[
-                    offset_row : offset_row + nb_row,
-                    offset_col : offset_col + nb_col,
-                ]
-            )
-
-        else:
-            nb_row = int(in_dataset[tag].values.shape[1])
-            nb_col = int(in_dataset[tag].values.shape[2])
-
-            values = np.ascontiguousarray(
-                full_data[
-                    :,
-                    offset_row : offset_row + nb_row,
-                    offset_col : offset_col + nb_col,
-                ]
-            )
-
-        xarray_coords = {}
-        for coord_tag in full_dataset[tag].coords:
-            if coord_tag in coords:
-                xarray_coords[coord_tag] = coords[coord_tag]
-            else:
-                xarray_coords[coord_tag] = full_dataset[tag].coords[coord_tag]
-
-        dims = full_dataset[tag].dims
-
-        data_array = xr.DataArray(values, coords=xarray_coords, dims=dims)
-        if tag in in_dataset:
-            data_array.attrs = in_dataset[tag].attrs
-        cropped[tag] = data_array
-
-    cropped.attrs = in_dataset.attrs
-
-    return cropped

cars/data_structures/format_transformation.py

@@ -30,9 +30,12 @@ import math
 import numpy as np
 import xarray as xr
 
+from cars.orchestrator.cluster.log_wrapper import cars_profile
+
 # CARS imports
 
 
+@cars_profile(name="Grid margins 2 overlaps", interval=0.5)
 def grid_margins_2_overlaps(grid, margins_fun):
     """
     Convert margins to overlap grid format used in CarsDatasets
@@ -48,7 +51,7 @@ def grid_margins_2_overlaps(grid, margins_fun):
 
     """
 
-    def fill_overlap(
+    def fill_overlap(  # pylint: disable=too-many-positional-arguments
         cars_ds_overlaps,
         margins,
         row_up,

cars/devibrate.py

@@ -95,7 +95,7 @@ def acquisition_direction(
     return time_direction_vector, vec1, vec2
 
 
-def get_time_ground_direction(
+def get_time_ground_direction(  # pylint: disable=too-many-positional-arguments
     sensor,
     geomodel,
     geometry_plugin,
@@ -180,6 +180,7 @@ def project_coordinates_on_line(
     return dist_to_origin * np.cos(proj_angle)
 
 
+# pylint: disable=too-many-positional-arguments
 def lowres_initial_dem_splines_fit(
     lowres_dsm_from_matches: xr.Dataset,
     lowres_initial_dem: xr.Dataset,
@@ -379,7 +380,7 @@ def read_lowres_dsm(srtm_path, startx, starty, endx, endy):
     return dsm_as_ds, newstartx, newstarty, sizex, sizey, resolution
 
 
-def compute_splines(
+def compute_splines(  # pylint: disable=too-many-positional-arguments
     sensor1,
     geomodel1,
     sensor2,
@@ -507,7 +508,7 @@ def compute_splines(
     }
 
 
-def cars_devibrate(
+def cars_devibrate(  # pylint: disable=too-many-positional-arguments
     used_conf,
     srtm_path,
     geoid_path,
@@ -594,6 +595,8 @@ def cars_devibrate(
         )
 
         x_values_2d, y_values_2d = np.meshgrid(x_values_1d, y_values_1d)
+
+        positions = None
         if src.crs != "EPSG:4326":
             transformer = pyproj.Transformer.from_crs(
                 src.crs, "EPSG:4326", always_xy=True

cars/extractroi.py

@@ -29,6 +29,7 @@ import numpy as np
 import rasterio as rio
 from affine import Affine
 from shapely.geometry import box
+from shareloc.geomodels.rpc_writers import write_rio_rpc_as_rpb
 
 
 def is_bbx_in_image(bbx, image_dataset):
@@ -75,7 +76,7 @@ def get_slices_from_bbx(image_dataset, bbx, rpc_options):
 
 
 def process_image_file(
-    bbx, input_image_path, output_image_path, geom_file_path, rpc_options
+    bbx, input_image_path, output_image_path, rpb_file_path, rpc_options
 ):
     """
     Processes an image file by extracting a region based on the given geometry.
@@ -84,7 +85,7 @@ def process_image_file(
         region_geometry (dict): GeoJSON-like dictionary defining the region.
         input_image_path (str): Path to the input image file.
         output_image_path (str): Path to save the output image.
-        geom_file_path (str): Path to save the .geom file.
+        rpb_file_path (str): Path to save the .RPB file.
         rpc_options (dict): Options for GDALCreateRPCTransformer.
     """
 
@@ -110,7 +111,8 @@ def process_image_file(
             # copy rpc
             dst.rpcs = image_dataset.rpcs
 
-        create_geom_file(image_dataset, geom_file_path)
+        if rpb_file_path is not None:
+            create_rpb_file(image_dataset, rpb_file_path)
 
 
 def get_human_readable_bbox(image_dataset, rpc_options):
@@ -167,34 +169,23 @@ def validate_bounding_box(bbx, image_dataset, rpc_options):
             image_dataset, rpc_options
         )
         raise ValueError(
-            f"Coordinates must between "
+            f"Coordinates must be between "
             f"({min_x}, {min_y}) and ({max_x}, {max_y})"
         )
 
 
-def create_geom_file(image_dataset, geom_filename):
+def create_rpb_file(image_dataset, rpb_filename):
     """
-    Create and save a .geom file from a rasterio dataset
+    Create and save a .RPB file from a rasterio dataset
 
     Parameters:
         image_dataset (rio.DatasetReader): Opened image dataset.
-        geom_filename (str): Path to save the .geom file.
+        rpb_filename (str): Path to save the .RPB file.
     """
     if not image_dataset.rpcs:
         raise ValueError("Image dataset has no RPCs")
     rpcs_as_dict = image_dataset.rpcs.to_dict()
-    with open(geom_filename, "w", encoding="utf-8") as writer:
-        for key in rpcs_as_dict:
-            if isinstance(rpcs_as_dict[key], list):
-                for idx, coef in enumerate(rpcs_as_dict[key]):
-                    writer.write(key + "_%02d" % idx + ": " + str(coef))
-                    writer.write("\n")
-            else:
-                writer.write(key + ": " + str(rpcs_as_dict[key]))
-                writer.write("\n")
-
-        writer.write("type:  ossimRpcModel\n")
-        writer.write("polynomial_format:  B\n")
+    write_rio_rpc_as_rpb(rpcs_as_dict, rpb_filename)
 
 
 def main():
@@ -240,6 +231,12 @@ def main():
         help="Digital Elevation Model used for projection",
     )
 
+    parser.add_argument(
+        "--generate_rpb",
+        action="store_true",
+        help="Generate RPB file",
+    )
+
     args = parser.parse_args()
     if not os.path.exists(args.out):
         os.makedirs(args.out)
@@ -254,10 +251,12 @@ def main():
     # check first input in list to determine pipeline
     for idx, image_path in enumerate(args.il):
         output_image_path = os.path.join(args.out, "ext_%03d.tif" % idx)
-        geom_file_path = os.path.splitext(output_image_path)[0] + ".geom"
+        rpb_file_path = None
+        if args.generate_rpb:
+            rpb_file_path = os.path.splitext(output_image_path)[0] + ".RPB"
 
         process_image_file(
-            args.bbx, image_path, output_image_path, geom_file_path, rpc_options
+            args.bbx, image_path, output_image_path, rpb_file_path, rpc_options
         )
 
 

cars/orchestrator/cluster/abstract_cluster.py

@@ -48,8 +48,13 @@ class AbstractCluster(metaclass=ABCMeta):
     # cluster mode output directory
     out_dir: str
 
-    def __new__(  # pylint: disable=W0613
-        cls, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __new__(  # pylint: disable=too-many-positional-arguments
+        cls,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):
         """
         Return the required cluster
@@ -98,8 +103,13 @@ class AbstractCluster(metaclass=ABCMeta):
 
         return decorator
 
-    def __init__(
-        self, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):  # pylint: disable=W0613
         """
         Init function of AbstractCluster
@@ -114,7 +124,7 @@ class AbstractCluster(metaclass=ABCMeta):
         # data to propagate
         self.data_to_propagate = data_to_propagate
 
-        self.worker_log_dir = os.path.join(out_dir, "logs", "workers_log")
+        self.worker_log_dir = os.path.join(log_dir, "workers_log")
         if not os.path.exists(self.worker_log_dir):
             os.makedirs(self.worker_log_dir)
 

cars/orchestrator/cluster/abstract_dask_cluster.py

@@ -56,7 +56,7 @@ class AbstractDaskCluster(
     AbstractDaskCluster
     """
 
-    def __init__(self, conf_cluster, out_dir, launch_worker=True):
+    def __init__(self, conf_cluster, out_dir, log_dir, launch_worker=True):
         """
         Init function of AbstractDaskCluster
 
@@ -64,8 +64,12 @@ class AbstractDaskCluster(
 
         """
 
+        print("cluster log_dir", log_dir)
+
         # call parent init
-        super().__init__(conf_cluster, out_dir, launch_worker=launch_worker)
+        super().__init__(
+            conf_cluster, out_dir, log_dir, launch_worker=launch_worker
+        )
         # retrieve parameters
         self.nb_workers = self.checked_conf_cluster["nb_workers"]
         self.task_timeout = self.checked_conf_cluster["task_timeout"]

cars/orchestrator/cluster/dask_jobqueue_utils.py

@@ -30,7 +30,7 @@ import warnings
 from datetime import timedelta
 
 
-def init_cluster_variables(
+def init_cluster_variables(  # pylint: disable=too-many-positional-arguments
     nb_workers,
     walltime,
     out_dir,