cars 0.12.0rc2__tar.gz → 1.0.0a1__tar.gz

cars-1.0.0a1/.gitlab-ci.yml

@@ -0,0 +1,161 @@
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_REF_NAME == "master"
+
+stages:
+  - init
+  - build
+  - test
+  - sonarqube
+  - sync
+
+
+variables: 
+    # Token used to push reports to SonarQube
+    SONARQUBE_TOKEN: "${CARS_CI_SONARQUBE_TOKEN}"
+
+    # Git variables
+    # Limit git depth to speed up cloning
+    GIT_DEPTH: 1
+
+include:
+  # Sonarqube job
+  - project: "usinelogicielle/public/gitlab-ci-templates" # Inclut le stage de qualité sonarqube
+    ref: v1.1.3 # référence de tag
+    file: "jobs/sonarqube-gitlab-ci.yml"
+  # Variable defintion
+  - component: $CI_SERVER_FQDN/dali/cars-park/cars-ci/cars-ci-variables@master
+  # Jobs for gitlab-github synchronisation
+  - component: $CI_SERVER_FQDN/dali/cars-park/cars-ci/cars-ci-github@master
+  # Docker building and image selection
+  # First case: no need to update env, use latest version
+  - local: '/ci/cars-no-docker-build.yml'
+    rules:
+      # do not trigger this case if env has changed(see second case)
+      - changes:
+        - "ci/cars-deps-env/Dockerfile"
+        - "pyproject.toml"
+        when: never
+      # do not trigger this case on master (see third case)
+      - if: $CI_COMMIT_REF_NAME == "master"
+        when: never
+      - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  # Second case: env should be generated on a dev branch, create a temporary
+  # image
+  - local: '/ci/cars-docker-build.yml'
+    inputs:
+      tag: ${CI_COMMIT_REF_NAME}
+    rules:
+      - changes:
+        - "ci/cars-deps-env/Dockerfile"
+        - "pyproject.toml"
+      # do not trigger this case on master (see third case)
+      - if: $CI_COMMIT_REF_NAME == "master"
+        when: never
+  # Third case: master branch pipeline: generate a "latest" image
+  - local: '/ci/cars-docker-build.yml'
+    inputs:
+      tag: "latest"
+    rules:
+      - if: $CI_COMMIT_REF_NAME == "master"
+
+# Common part for jobs using python: use env image and install cars
+.cars-python-job:
+  image: ${UL_ARTIFACTORY_HOST}/${CARS_ENV_IMAGE_NAME}:${CARS_ENV_IMAGE_TAG}
+  before_script:
+    - source /app/cars/venv/bin/activate
+    # All dependencies should already be installed in the docker env
+    - pip install --no-build-isolation --editable .[dev,docs,notebook]
+
+
+# run unit tests before other tasks (lint, quality, end2end ...) to catch early
+# errors
+unit-test:
+  stage: test
+  extends:
+    - .cars-python-job
+  script:
+    - CARS_VENV="/app/cars/venv" make test/ci
+  artifacts:
+    paths:
+      - ./pytest-report.xml
+      - ./coverage.xml
+    expire_in: 1 day
+
+
+# Job testing that notebooks run properly in Python
+test-notebooks:
+  stage: test
+  needs:
+    - unit-test
+  extends:
+    - .cars-python-job
+  script:
+    # Install additional plugin dependencies required for the tests
+    - source /app/cars/venv/bin/activate
+    - pip install --no-build-isolation --editable .[pandora_mccnn,bundleadjustment]
+    - CARS_VENV="/app/cars/venv" make test/notebook
+
+# test on full cars pipelines
+test-end2end:
+  stage: test
+  needs:
+    - unit-test
+  # Use dedicated runner for end2end tests
+  tags:
+    - ${TEST_RUNNER_TAG}
+  timeout: 80 minutes 
+  extends:
+    - .cars-python-job
+  script:
+    - source /app/cars/venv/bin/activate
+    - pytest -m "end2end_tests"
+  artifacts:
+    paths:
+      - tests/data/intermediate_data
+    when: always
+    expire_in: 1 day
+
+
+
+# All lint tasks are performed in the same job to save up on job initialization
+# time
+lint:
+  stage: test
+  needs:
+    - unit-test
+  extends:
+    - .cars-python-job
+  allow_failure: true
+  script:
+    - CARS_VENV="/app/cars/venv" make lint
+  artifacts:
+    paths:
+      - ./pylint-report.txt
+    expire_in: 1 day
+
+
+doc:
+  stage: test
+  needs:
+    - unit-test
+  extends:
+    - .cars-python-job
+  script:
+    - CARS_VENV="/app/cars/venv" make docs
+  artifacts:
+    # Export documentation in artifact, it can be opened directly from gitlab
+    paths:
+      - ./docs/build/html/
+    expire_in: 1 day
+
+
+# Push git project from gitlab to github
+github-sync:
+  extends:
+    - .github-sync-base
+
+  variables:
+    BRANCH: "master"
+    GITHUB_URL: github.com/CNES/cars.git

{cars-0.12.0rc2 → cars-1.0.0a1}/AUTHORS.md

@@ -38,7 +38,7 @@ This file keeps track of authors contributions.
 * Alice de Bardonneche-Richard <alice.de-bardonneche-richard@csgroup.eu>
 * Tommy Calendini <tommy.calendini@cs-soprasteria.com>
 * Dimitri Lallement <dimitri.lallement@cnes.fr>
-* Datafalk <louis@datafalk.com>
+* Datafalk <research@datafalk.com>
 * Marian Rassat <marian.rassat@cs-soprasteria.com>
 
 Update here with new contributors.

{cars-0.12.0rc2 → cars-1.0.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cars
-Version: 0.12.0rc2
+Version: 1.0.0a1
 Summary: A satellite multi view stereo pipeline
 Keywords: cars,3D,DEM,pandora,photogrammetry
 Author-Email: CNES <cars@cnes.fr>
@@ -38,7 +38,7 @@ Requires-Dist: urllib3<2.0,>=1.21.1
 Requires-Dist: laspy[laszip]
 Requires-Dist: tbb==2020.3.254
 Requires-Dist: numba
-Requires-Dist: pandora[sgm]==1.7.0
+Requires-Dist: pandora[sgm]==1.7.1a1
 Requires-Dist: cars-rasterize==0.2.*
 Requires-Dist: cars-resample==0.1.*
 Requires-Dist: cars-filter==0.2.*

{cars-0.12.0rc2 → cars-1.0.0a1}/cars/applications/auxiliary_filling/__init__.py

@@ -23,7 +23,7 @@ CARS auxiliary filling module init file
 """
 # flake8: noqa: F401
 
-import cars.applications.auxiliary_filling.auxiliary_filling
-from cars.applications.auxiliary_filling.auxiliary_filling_from_sensors import (
+import cars.applications.auxiliary_filling.abstract_auxiliary_filling_app
+from cars.applications.auxiliary_filling.auxiliary_filling_from_sensors_app import (  # pylint: disable=C0301
     AuxiliaryFilling,
 )

cars-0.12.0rc2/cars/applications/auxiliary_filling/auxiliary_filling_tools.py → cars-1.0.0a1/cars/applications/auxiliary_filling/auxiliary_filling_algo.py

@@ -26,9 +26,6 @@ this module contains the AuxiliaryFillingFromSensors application class.
 import numpy as np
 import rasterio as rio
 from scipy import interpolate
-from shapely.geometry import Polygon
-
-from cars.core.projection import polygon_projection
 
 
 def fill_auxiliary(
@@ -40,7 +37,8 @@ def fill_auxiliary(
     geom_plugin,
     number_of_color_bands,
     number_of_classification_bands,
-    color_interpolator,
+    texture_bands,
+    texture_interpolator,
     use_mask=False,
 ):
     """
@@ -63,9 +61,11 @@ def fill_auxiliary(
     :type number_of_color_bands: int
     :param number_of_classification_bands: number of bands in the color image
     :type number_of_classification_bands: int
-    :param color_interpolator: scipy interpolator use to interpolate color
+    :param texture_bands: list of band names used for output texture
+    :type texture_bands: list
+    :param texture_interpolator: scipy interpolator use to interpolate color
         values
-    :type color_interpolator: str
+    :type texture_interpolator: str
     :param use_mask: use mask information from sensors in color computation
     :type use_mask: bool
 
@@ -107,7 +107,8 @@ def fill_auxiliary(
                 geom_plugin,
                 number_of_color_bands,
                 number_of_classification_bands,
-                color_interpolator,
+                texture_bands,
+                texture_interpolator,
                 not_interpolated_mask=None,
                 use_mask=use_mask,
                 return_all_points=True,
@@ -132,7 +133,8 @@ def fill_auxiliary(
                     geom_plugin,
                     number_of_color_bands,
                     number_of_classification_bands,
-                    color_interpolator,
+                    texture_bands,
+                    texture_interpolator,
                     not_interpolated_mask,
                     use_mask=use_mask,
                     return_all_points=False,
@@ -165,7 +167,8 @@ def fill_from_one_sensor(  # noqa C901
     geom_plugin,
     number_of_color_bands,
     number_of_classification_bands,
-    color_interpolator,
+    texture_bands,
+    texture_interpolator,
     not_interpolated_mask=None,
     use_mask=False,
     return_all_points=False,
@@ -194,9 +197,11 @@ def fill_from_one_sensor(  # noqa C901
     :type number_of_color_bands: int
     :param number_of_classification_bands: number of bands in the color image
     :type number_of_classification_bands: int
-    :param color_interpolator: scipy interpolator use to interpolate color
+    :param texture_bands: list of band names used for output texture
+    :type texture_bands: list
+    :param texture_interpolator: scipy interpolator use to interpolate color
         values
-    :type color_interpolator: str
+    :type texture_interpolator: str
     :param not_interpolated_mask: use mask information in color computation
     :type not_interpolated_mask: numpy.array
     :param use_mask: use mask information in color computation
@@ -207,9 +212,7 @@ def fill_from_one_sensor(  # noqa C901
     """
 
     # Check if the sensor has color or classification
-    reference_sensor_image = sensor.get(
-        "color", sensor.get("classification", None)
-    )
+    reference_sensor_image = sensor["image"]["main_file"]
 
     output_not_interpolated_mask = np.ones(len(altitudes), dtype=bool)
     all_values = np.zeros((number_of_color_bands, len(altitudes)))
@@ -251,12 +254,12 @@ def fill_from_one_sensor(  # noqa C901
     ):
         return output_not_interpolated_mask, all_values
 
-    if color_interpolator in ("linear", "nearest"):
-        color_interpolator_margin = 1
-    elif color_interpolator == "cubic":
-        color_interpolator_margin = 3
+    if texture_interpolator in ("linear", "nearest"):
+        texture_interpolator_margin = 1
+    elif texture_interpolator == "cubic":
+        texture_interpolator_margin = 3
     else:
-        raise RuntimeError(f"Invalid interpolator {color_interpolator}")
+        raise RuntimeError(f"Invalid interpolator {texture_interpolator}")
 
     # Classification interpolator is always nearest
     classif_interpolator = "nearest"
@@ -308,100 +311,112 @@ def fill_from_one_sensor(  # noqa C901
                     validity_mask, not_interpolated_mask
                 )
 
-    if sensor.get("color"):
-        with rio.open(sensor["color"]) as sensor_color_image:
-            # Only fill color if the number of bands matches. For exemple do
-            # not try to fill an output RGB image from a greyscale "color"
-            # image.
-            if sensor_color_image.count == number_of_color_bands:
+    if sensor.get("image"):
+        # Only fill color if all texture bands are present
+        if all(
+            band_name in sensor["image"]["bands"] for band_name in texture_bands
+        ):
+            with rio.open(sensor["image"]["main_file"]) as sensor_color_image:
                 first_row = np.floor(
-                    max(np.min(ind_rows_sensor) - color_interpolator_margin, 0)
+                    max(
+                        np.min(ind_rows_sensor) - texture_interpolator_margin, 0
+                    )
                 )
                 last_row = np.ceil(
                     min(
-                        np.max(ind_rows_sensor) + color_interpolator_margin,
+                        np.max(ind_rows_sensor) + texture_interpolator_margin,
                         sensor_color_image.height,
                     )
                 )
                 first_col = np.floor(
-                    max(np.min(ind_cols_sensor) - color_interpolator_margin, 0)
+                    max(
+                        np.min(ind_cols_sensor) - texture_interpolator_margin, 0
+                    )
                 )
                 last_col = np.ceil(
                     min(
-                        np.max(ind_cols_sensor) + color_interpolator_margin,
+                        np.max(ind_cols_sensor) + texture_interpolator_margin,
                         sensor_color_image.width,
                     )
                 )
 
-                rio_window = rio.windows.Window.from_slices(
-                    (first_row, last_row),
-                    (first_col, last_col),
-                )
+            rio_window = rio.windows.Window.from_slices(
+                (first_row, last_row),
+                (first_col, last_col),
+            )
 
-                sensor_points = (
-                    np.arange(first_row, last_row),
-                    np.arange(first_col, last_col),
-                )
+            sensor_points = (
+                np.arange(first_row, last_row),
+                np.arange(first_col, last_col),
+            )
 
-                if validity_mask is not None:
-                    interpolated_mask = validity_mask
-                else:
-                    interpolated_mask = np.ones(len(altitudes), dtype=bool)
+            if validity_mask is not None:
+                interpolated_mask = validity_mask
+            else:
+                interpolated_mask = np.ones(len(altitudes), dtype=bool)
 
-                for band in range(sensor_color_image.count):
-                    # rio band convention
-                    sensor_data = sensor_color_image.read(
-                        band + 1, window=rio_window
-                    )
+            for output_band, band_name in enumerate(texture_bands):
+                # rio band convention
+                sensor_file = rio.open(
+                    sensor["image"]["bands"][band_name]["path"]
+                )
+                input_band = sensor["image"]["bands"][band_name]["band"]
+                sensor_data = sensor_file.read(
+                    input_band + 1, window=rio_window
+                )
 
-                    if validity_mask is not None:
-                        if return_all_points is True:
-                            all_values[band, :] = interpolate.interpn(
-                                sensor_points,
-                                sensor_data,
-                                (ind_rows_sensor, ind_cols_sensor),
-                                bounds_error=False,
-                                method=color_interpolator,
-                            )
-                            band_values = all_values[band, validity_mask]
-                        # No need to interpolate on every points
-                        else:
-                            band_values = interpolate.interpn(
-                                sensor_points,
-                                sensor_data,
-                                (
-                                    ind_rows_sensor[validity_mask],
-                                    ind_cols_sensor[validity_mask],
-                                ),
-                                bounds_error=False,
-                                method=color_interpolator,
-                            )
-                        nan_values = np.isnan(band_values)
-                        interpolated_mask[validity_mask] = np.logical_or(
-                            interpolated_mask[validity_mask], ~nan_values
+                if validity_mask is not None:
+                    if return_all_points is True:
+                        all_values[output_band, :] = interpolate.interpn(
+                            sensor_points,
+                            sensor_data,
+                            (ind_rows_sensor, ind_cols_sensor),
+                            bounds_error=False,
+                            method=texture_interpolator,
                         )
-                        filled_color[band, interpolated_mask] += band_values
-
+                        band_values = all_values[output_band, validity_mask]
+                    # No need to interpolate on every points
                     else:
                         band_values = interpolate.interpn(
                             sensor_points,
                             sensor_data,
-                            (ind_rows_sensor, ind_cols_sensor),
+                            (
+                                ind_rows_sensor[validity_mask],
+                                ind_cols_sensor[validity_mask],
+                            ),
                             bounds_error=False,
-                            method=color_interpolator,
+                            method=texture_interpolator,
                         )
-                        interpolated_mask = np.logical_or(
-                            interpolated_mask, ~np.isnan(band_values)
-                        )
-                        filled_color[band, interpolated_mask] += band_values
-                output_not_interpolated_mask = ~interpolated_mask
+                    nan_values = np.isnan(band_values)
+                    interpolated_mask[validity_mask] = np.logical_or(
+                        interpolated_mask[validity_mask], ~nan_values
+                    )
+                    filled_color[output_band, interpolated_mask] += band_values
 
-                weights[interpolated_mask] += 1
+                else:
+                    band_values = interpolate.interpn(
+                        sensor_points,
+                        sensor_data,
+                        (ind_rows_sensor, ind_cols_sensor),
+                        bounds_error=False,
+                        method=texture_interpolator,
+                    )
+                    interpolated_mask = np.logical_or(
+                        interpolated_mask, ~np.isnan(band_values)
+                    )
+                    filled_color[output_band, interpolated_mask] += band_values
+            output_not_interpolated_mask = ~interpolated_mask
+
+            weights[interpolated_mask] += 1
 
     if filled_classif is not None and sensor.get("classification"):
-        with rio.open(sensor["classification"]) as sensor_classif_image:
+        if number_of_classification_bands == len(
+            sensor["classification"]["bands"]
+        ):
+            with rio.open(
+                sensor["classification"]["main_file"]
+            ) as sensor_classif_image:
 
-            if sensor_classif_image.count == number_of_classification_bands:
                 first_row = np.floor(
                     max(
                         np.min(ind_rows_sensor) - classif_interpolator_margin, 0
@@ -425,95 +440,39 @@ def fill_from_one_sensor(  # noqa C901
                     )
                 )
 
-                rio_window = rio.windows.Window.from_slices(
-                    (first_row, last_row),
-                    (first_col, last_col),
-                )
+            rio_window = rio.windows.Window.from_slices(
+                (first_row, last_row),
+                (first_col, last_col),
+            )
 
-                sensor_points = (
-                    np.arange(first_row, last_row),
-                    np.arange(first_col, last_col),
-                )
+            sensor_points = (
+                np.arange(first_row, last_row),
+                np.arange(first_col, last_col),
+            )
 
-                for band in range(sensor_classif_image.count):
-                    # rio band convention
-                    sensor_data = sensor_classif_image.read(
-                        band + 1, window=rio_window, out_dtype=np.uint8
-                    )
+            for output_band, band_name in enumerate(
+                sensor["classification"]["bands"]
+            ):
+                # rio band convention
+                sensor_file = rio.open(
+                    sensor["classification"]["bands"][band_name]["path"]
+                )
+                input_band = sensor["classification"]["bands"][band_name][
+                    "band"
+                ]
+                sensor_data = sensor_file.read(
+                    input_band + 1, window=rio_window
+                )
 
-                    filled_classif[band, :] = np.logical_or(
-                        filled_classif[band, :],
-                        interpolate.interpn(
-                            sensor_points,
-                            sensor_data,
-                            (ind_rows_sensor, ind_cols_sensor),
-                            bounds_error=False,
-                            method=classif_interpolator,
-                        ),
-                    )
+                filled_classif[output_band, :] = np.logical_or(
+                    filled_classif[output_band, :],
+                    interpolate.interpn(
+                        sensor_points,
+                        sensor_data,
+                        (ind_rows_sensor, ind_cols_sensor),
+                        bounds_error=False,
+                        method=classif_interpolator,
+                    ),
+                )
 
     return output_not_interpolated_mask, all_values
-
-
-def compute_sensor_bounds(sensor_inputs, geom_plugin, output_epsg):
-    """
-    Compute bounds of each input sensor that have an associated color or
-    classification image
-
-    :param sensor_inputs: dictionary containing paths to input images and models
-    :type sensor_inputs: dict
-    :param geom_plugin: geometry plugin used for inverse locations
-    :type geom_plugin: AbstractGeometry
-    :param geom_plugin: geometry plugin used for inverse locations
-    :type geom_plugin: AbstractGeometry
-    :param output_epsg: epsg of the output polygons
-    :type output_epsg: int
-
-    :return: a dictionary containing a Polygon in output geometry for each
-        valid input sensor
-    """
-
-    sensor_bounds = {}
-
-    for sensor_name, sensor in sensor_inputs.items():
-        reference_sensor_image = sensor.get(
-            "color", sensor.get("classification", None)
-        )
-        # no data for this sensor, no need to compute its bounds
-        if reference_sensor_image is None:
-            continue
-
-        u_l, u_r, l_r, l_l = geom_plugin.image_envelope(
-            reference_sensor_image, sensor["geomodel"]
-        )
-
-        poly_geo = Polygon([u_l, u_r, l_r, l_l, u_l])
-
-        sensor_bounds[sensor_name] = polygon_projection(
-            poly_geo, 4326, output_epsg
-        )
-
-    return sensor_bounds
-
-
-def filter_sensor_inputs(sensor_inputs, sensor_bounds, ground_polygon):
-    """
-    Filter input sensors by comparing their bounds to a reference Polygon
-
-    :param sensor_inputs: dictionary containing paths to input images and models
-    :type sensor_inputs: dict
-    :param sensor_bounds: dictionary containing bounds of input sensors
-    :type sensor_bounds: dict
-    :param ground_polygon: reference polygon, in ground geometry
-    :type ground_polygon: Polygon
-
-    :return: a fitlered version of sensor_inputs
-    """
-
-    filtered_sensor_inputs = sensor_inputs.copy()
-
-    for sensor_name, bound in sensor_bounds.items():
-        if not bound.intersects(ground_polygon):
-            del filtered_sensor_inputs[sensor_name]
-
-    return filtered_sensor_inputs