cars 1.0.0a2__cp310-cp310-win_amd64.whl → 1.0.0a4__cp310-cp310-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cars might be problematic. Click here for more details.

Files changed (144) hide show
  1. cars/__init__.py +3 -3
  2. cars/applications/__init__.py +0 -3
  3. cars/applications/application.py +14 -6
  4. cars/applications/application_template.py +42 -0
  5. cars/applications/auxiliary_filling/abstract_auxiliary_filling_app.py +12 -2
  6. cars/applications/auxiliary_filling/auxiliary_filling_algo.py +2 -2
  7. cars/applications/auxiliary_filling/auxiliary_filling_from_sensors_app.py +95 -46
  8. cars/applications/auxiliary_filling/auxiliary_filling_wrappers.py +7 -6
  9. cars/applications/dem_generation/abstract_dem_generation_app.py +9 -5
  10. cars/applications/dem_generation/dem_generation_algo.py +1 -1
  11. cars/applications/dem_generation/dem_generation_wrappers.py +44 -59
  12. cars/applications/dem_generation/dichotomic_generation_app.py +9 -6
  13. cars/applications/dem_generation/rasterization_app.py +112 -43
  14. cars/applications/dense_match_filling/__init__.py +1 -1
  15. cars/applications/dense_match_filling/abstract_dense_match_filling_app.py +2 -15
  16. cars/applications/dense_match_filling/fill_disp_algo.py +32 -373
  17. cars/applications/dense_match_filling/fill_disp_wrappers.py +0 -343
  18. cars/applications/dense_match_filling/zero_padding_app.py +10 -5
  19. cars/applications/dense_matching/abstract_dense_matching_app.py +2 -1
  20. cars/applications/dense_matching/census_mccnn_sgm_app.py +48 -60
  21. cars/applications/dense_matching/cpp/dense_matching_cpp.cp310-win_amd64.dll.a +0 -0
  22. cars/applications/dense_matching/cpp/dense_matching_cpp.cp310-win_amd64.pyd +0 -0
  23. cars/applications/dense_matching/dense_matching_algo.py +48 -14
  24. cars/applications/dense_matching/dense_matching_wrappers.py +11 -3
  25. cars/applications/dense_matching/disparity_grid_algo.py +95 -79
  26. cars/applications/dense_matching/loaders/config_mapping.json +13 -0
  27. cars/applications/dense_matching/loaders/global_land_cover_map.tif +0 -0
  28. cars/applications/dense_matching/loaders/pandora_loader.py +169 -34
  29. cars/applications/dsm_filling/border_interpolation_app.py +11 -12
  30. cars/applications/dsm_filling/bulldozer_filling_app.py +16 -15
  31. cars/applications/dsm_filling/exogenous_filling_app.py +14 -14
  32. cars/applications/grid_generation/abstract_grid_generation_app.py +1 -1
  33. cars/applications/grid_generation/epipolar_grid_generation_app.py +4 -2
  34. cars/applications/grid_generation/grid_correction_app.py +4 -1
  35. cars/applications/grid_generation/grid_generation_algo.py +7 -2
  36. cars/applications/ground_truth_reprojection/abstract_ground_truth_reprojection_app.py +1 -1
  37. cars/applications/ground_truth_reprojection/direct_localization_app.py +2 -2
  38. cars/applications/ground_truth_reprojection/ground_truth_reprojection_algo.py +2 -1
  39. cars/applications/point_cloud_fusion/abstract_pc_fusion_app.py +0 -155
  40. cars/applications/point_cloud_fusion/mapping_to_terrain_tiles_app.py +0 -658
  41. cars/applications/point_cloud_fusion/pc_fusion_algo.py +0 -1339
  42. cars/applications/point_cloud_fusion/pc_fusion_wrappers.py +0 -869
  43. cars/applications/point_cloud_outlier_removal/abstract_outlier_removal_app.py +11 -6
  44. cars/applications/point_cloud_outlier_removal/outlier_removal_algo.py +9 -8
  45. cars/applications/point_cloud_outlier_removal/small_components_app.py +101 -270
  46. cars/applications/point_cloud_outlier_removal/statistical_app.py +120 -277
  47. cars/applications/rasterization/abstract_pc_rasterization_app.py +2 -1
  48. cars/applications/rasterization/rasterization_algo.py +18 -6
  49. cars/applications/rasterization/rasterization_wrappers.py +2 -1
  50. cars/applications/rasterization/simple_gaussian_app.py +88 -116
  51. cars/applications/resampling/abstract_resampling_app.py +1 -1
  52. cars/applications/resampling/bicubic_resampling_app.py +3 -1
  53. cars/applications/resampling/resampling_algo.py +60 -53
  54. cars/applications/resampling/resampling_wrappers.py +3 -1
  55. cars/applications/sparse_matching/abstract_sparse_matching_app.py +1 -1
  56. cars/applications/sparse_matching/sift_app.py +5 -25
  57. cars/applications/sparse_matching/sparse_matching_algo.py +3 -2
  58. cars/applications/sparse_matching/sparse_matching_wrappers.py +1 -1
  59. cars/applications/triangulation/abstract_triangulation_app.py +1 -1
  60. cars/applications/triangulation/line_of_sight_intersection_app.py +13 -11
  61. cars/applications/triangulation/pc_transform.py +552 -0
  62. cars/applications/triangulation/triangulation_algo.py +6 -4
  63. cars/applications/triangulation/triangulation_wrappers.py +1 -0
  64. cars/bundleadjustment.py +6 -6
  65. cars/cars.py +11 -9
  66. cars/core/cars_logging.py +80 -49
  67. cars/core/constants.py +0 -1
  68. cars/core/datasets.py +5 -2
  69. cars/core/geometry/abstract_geometry.py +364 -22
  70. cars/core/geometry/shareloc_geometry.py +112 -82
  71. cars/core/inputs.py +72 -19
  72. cars/core/outputs.py +1 -1
  73. cars/core/preprocessing.py +17 -3
  74. cars/core/projection.py +126 -6
  75. cars/core/tiling.py +10 -3
  76. cars/data_structures/cars_dataset.py +12 -10
  77. cars/data_structures/corresponding_tiles_tools.py +0 -103
  78. cars/data_structures/format_transformation.py +4 -1
  79. cars/devibrate.py +6 -3
  80. cars/extractroi.py +20 -21
  81. cars/orchestrator/cluster/abstract_cluster.py +15 -5
  82. cars/orchestrator/cluster/abstract_dask_cluster.py +6 -2
  83. cars/orchestrator/cluster/dask_jobqueue_utils.py +1 -1
  84. cars/orchestrator/cluster/log_wrapper.py +149 -22
  85. cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +12 -4
  86. cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py +2 -2
  87. cars/orchestrator/cluster/pbs_dask_cluster.py +1 -1
  88. cars/orchestrator/cluster/sequential_cluster.py +5 -4
  89. cars/orchestrator/cluster/slurm_dask_cluster.py +1 -1
  90. cars/orchestrator/orchestrator.py +15 -4
  91. cars/orchestrator/registry/id_generator.py +1 -0
  92. cars/orchestrator/registry/saver_registry.py +2 -2
  93. cars/pipelines/conf_resolution/conf_final_resolution.json +5 -3
  94. cars/pipelines/default/default_pipeline.py +461 -1052
  95. cars/pipelines/parameters/advanced_parameters.py +91 -64
  96. cars/pipelines/parameters/advanced_parameters_constants.py +6 -5
  97. cars/pipelines/parameters/application_parameters.py +71 -0
  98. cars/pipelines/parameters/depth_map_inputs.py +0 -314
  99. cars/pipelines/parameters/dsm_inputs.py +40 -4
  100. cars/pipelines/parameters/output_parameters.py +44 -8
  101. cars/pipelines/parameters/sensor_inputs.py +122 -73
  102. cars/pipelines/parameters/sensor_inputs_constants.py +0 -2
  103. cars/pipelines/parameters/sensor_loaders/__init__.py +4 -3
  104. cars/pipelines/parameters/sensor_loaders/basic_classif_loader.py +106 -0
  105. cars/pipelines/parameters/sensor_loaders/{basic_sensor_loader.py → basic_image_loader.py} +16 -22
  106. cars/pipelines/parameters/sensor_loaders/pivot_classif_loader.py +121 -0
  107. cars/pipelines/parameters/sensor_loaders/{pivot_sensor_loader.py → pivot_image_loader.py} +10 -21
  108. cars/pipelines/parameters/sensor_loaders/sensor_loader.py +4 -6
  109. cars/pipelines/parameters/sensor_loaders/sensor_loader_template.py +1 -3
  110. cars/pipelines/pipeline_template.py +1 -3
  111. cars/pipelines/unit/unit_pipeline.py +676 -1070
  112. cars/starter.py +4 -3
  113. cars-1.0.0a4.dist-info/DELVEWHEEL +2 -0
  114. {cars-1.0.0a2.dist-info → cars-1.0.0a4.dist-info}/METADATA +135 -53
  115. {cars-1.0.0a2.dist-info → cars-1.0.0a4.dist-info}/RECORD +120 -134
  116. cars.libs/libgcc_s_seh-1-b2494fcbd4d80cf2c98fdd5261f6d850.dll +0 -0
  117. cars.libs/libstdc++-6-e9b0d12ae0e9555bbae55e8dfd08c3f7.dll +0 -0
  118. cars.libs/libwinpthread-1-7882d1b093714ccdfaf4e0789a817792.dll +0 -0
  119. cars/applications/dense_match_filling/cpp/__init__.py +0 -0
  120. cars/applications/dense_match_filling/cpp/dense_match_filling_cpp.cp310-win_amd64.dll.a +0 -0
  121. cars/applications/dense_match_filling/cpp/dense_match_filling_cpp.cp310-win_amd64.pyd +0 -0
  122. cars/applications/dense_match_filling/cpp/dense_match_filling_cpp.py +0 -72
  123. cars/applications/dense_match_filling/cpp/includes/dense_match_filling.hpp +0 -46
  124. cars/applications/dense_match_filling/cpp/meson.build +0 -9
  125. cars/applications/dense_match_filling/cpp/src/bindings.cpp +0 -11
  126. cars/applications/dense_match_filling/cpp/src/dense_match_filling.cpp +0 -142
  127. cars/applications/dense_match_filling/plane_app.py +0 -556
  128. cars/applications/hole_detection/__init__.py +0 -30
  129. cars/applications/hole_detection/abstract_hole_detection_app.py +0 -125
  130. cars/applications/hole_detection/cloud_to_bbox_app.py +0 -346
  131. cars/applications/hole_detection/hole_detection_algo.py +0 -144
  132. cars/applications/hole_detection/hole_detection_wrappers.py +0 -53
  133. cars/applications/point_cloud_denoising/__init__.py +0 -29
  134. cars/applications/point_cloud_denoising/abstract_pc_denoising_app.py +0 -273
  135. cars/applications/point_cloud_fusion/__init__.py +0 -30
  136. cars/applications/point_cloud_fusion/cloud_fusion_constants.py +0 -39
  137. cars/applications/sparse_matching/pandora_sparse_matching_app.py +0 -0
  138. cars/pipelines/parameters/depth_map_inputs_constants.py +0 -25
  139. cars-1.0.0a2.dist-info/DELVEWHEEL +0 -2
  140. cars.libs/libgcc_s_seh-1-f2b6825d483bdf14050493af93b5997d.dll +0 -0
  141. cars.libs/libstdc++-6-6b0059df6bc601df5a0f18a5805eea05.dll +0 -0
  142. cars.libs/libwinpthread-1-e01b8e85fd67c2b861f64d4ccc7df607.dll +0 -0
  143. {cars-1.0.0a2.dist-info → cars-1.0.0a4.dist-info}/WHEEL +0 -0
  144. {cars-1.0.0a2.dist-info → cars-1.0.0a4.dist-info}/entry_points.txt +0 -0
@@ -32,10 +32,10 @@ from __future__ import print_function
32
32
 
33
33
  import copy
34
34
  import logging
35
- import math
36
35
  import os
37
36
 
38
37
  import numpy as np
38
+ from pyproj import CRS
39
39
 
40
40
  import cars.applications.sparse_matching.sparse_matching_constants as sm_cst
41
41
  from cars import __version__
@@ -48,14 +48,11 @@ from cars.applications.dem_generation import (
48
48
  )
49
49
  from cars.applications.grid_generation import grid_correction_app
50
50
  from cars.applications.grid_generation.transform_grid import transform_grid_func
51
- from cars.applications.point_cloud_fusion import (
52
- pc_fusion_algo,
53
- pc_fusion_wrappers,
54
- )
55
51
  from cars.core import preprocessing, projection, roi_tools
56
52
  from cars.core.geometry.abstract_geometry import AbstractGeometry
57
53
  from cars.core.inputs import (
58
54
  get_descriptions_bands,
55
+ rasterio_get_crs,
59
56
  rasterio_get_epsg,
60
57
  rasterio_get_size,
61
58
  read_vector,
@@ -66,13 +63,14 @@ from cars.orchestrator import orchestrator
66
63
  from cars.orchestrator.cluster.log_wrapper import cars_profile
67
64
  from cars.pipelines.parameters import advanced_parameters
68
65
  from cars.pipelines.parameters import advanced_parameters_constants as adv_cst
69
- from cars.pipelines.parameters import depth_map_inputs
70
- from cars.pipelines.parameters import depth_map_inputs_constants as depth_cst
71
- from cars.pipelines.parameters import dsm_inputs
66
+ from cars.pipelines.parameters import application_parameters, dsm_inputs
72
67
  from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst
73
68
  from cars.pipelines.parameters import output_constants as out_cst
74
69
  from cars.pipelines.parameters import output_parameters, sensor_inputs
75
70
  from cars.pipelines.parameters import sensor_inputs_constants as sens_cst
71
+ from cars.pipelines.parameters.advanced_parameters_constants import (
72
+ USE_ENDOGENOUS_DEM,
73
+ )
76
74
  from cars.pipelines.pipeline import Pipeline
77
75
  from cars.pipelines.pipeline_constants import (
78
76
  ADVANCED,
@@ -107,7 +105,6 @@ class UnitPipeline(PipelineTemplate):
107
105
  save_output_point_clouds
108
106
  geom_plugin_without_dem_and_geoid
109
107
  geom_plugin_with_dem_and_geoid
110
- dem_generation_roi
111
108
 
112
109
  :param pipeline_name: name of the pipeline.
113
110
  :type pipeline_name: str
@@ -119,6 +116,8 @@ class UnitPipeline(PipelineTemplate):
119
116
 
120
117
  # Used conf
121
118
  self.used_conf = {}
119
+ # refined conf
120
+ self.refined_conf = {}
122
121
 
123
122
  # Transform relative path to absolute path
124
123
  if config_dir is not None:
@@ -135,21 +134,35 @@ class UnitPipeline(PipelineTemplate):
135
134
  # Check conf inputs
136
135
  inputs = self.check_inputs(conf[INPUTS], config_dir=config_dir)
137
136
  self.used_conf[INPUTS] = inputs
137
+ self.refined_conf[INPUTS] = copy.deepcopy(inputs)
138
138
 
139
139
  # Check advanced parameters
140
140
  # TODO static method in the base class
141
+ output_dem_dir = os.path.join(
142
+ conf[OUTPUT][out_cst.OUT_DIRECTORY], "dump_dir", "initial_elevation"
143
+ )
144
+ safe_makedirs(output_dem_dir)
141
145
  (
142
146
  inputs,
143
147
  advanced,
144
148
  self.geometry_plugin,
145
149
  self.geom_plugin_without_dem_and_geoid,
146
150
  self.geom_plugin_with_dem_and_geoid,
147
- self.dem_generation_roi,
151
+ self.scaling_coeff,
152
+ self.land_cover_map,
153
+ self.classification_to_config_mapping,
148
154
  ) = advanced_parameters.check_advanced_parameters(
149
- inputs, conf.get(ADVANCED, {}), check_epipolar_a_priori=True
155
+ inputs,
156
+ conf.get(ADVANCED, {}),
157
+ check_epipolar_a_priori=True,
158
+ output_dem_dir=output_dem_dir,
150
159
  )
151
160
  self.used_conf[ADVANCED] = advanced
152
161
 
162
+ self.refined_conf[ADVANCED] = copy.deepcopy(advanced)
163
+ # Refined conf: resolutions 1
164
+ self.refined_conf[ADVANCED][adv_cst.EPIPOLAR_RESOLUTIONS] = [1]
165
+
153
166
  # Get ROI
154
167
  (
155
168
  self.input_roi_poly,
@@ -161,8 +174,16 @@ class UnitPipeline(PipelineTemplate):
161
174
  self.debug_with_roi = self.used_conf[ADVANCED][adv_cst.DEBUG_WITH_ROI]
162
175
 
163
176
  # Check conf output
164
- output = self.check_output(conf[OUTPUT])
177
+ (
178
+ output,
179
+ self.scaling_coeff,
180
+ ) = self.check_output(conf[OUTPUT], self.scaling_coeff)
181
+
165
182
  self.used_conf[OUTPUT] = output
183
+ self.out_dir = self.used_conf[OUTPUT][out_cst.OUT_DIRECTORY]
184
+ self.dump_dir = os.path.join(self.out_dir, "dump_dir")
185
+
186
+ self.refined_conf[OUTPUT] = copy.deepcopy(output)
166
187
 
167
188
  prod_level = output[out_cst.PRODUCT_LEVEL]
168
189
 
@@ -177,11 +198,7 @@ class UnitPipeline(PipelineTemplate):
177
198
  or self.save_output_point_cloud
178
199
  )
179
200
  self.sensors_in_inputs = sens_cst.SENSORS in self.used_conf[INPUTS]
180
- self.depth_maps_in_inputs = (
181
- depth_cst.DEPTH_MAPS in self.used_conf[INPUTS]
182
- )
183
201
  self.dsms_in_inputs = dsm_cst.DSMS in self.used_conf[INPUTS]
184
- self.merging = self.used_conf[ADVANCED][adv_cst.MERGING]
185
202
 
186
203
  self.phasing = self.used_conf[ADVANCED][adv_cst.PHASING]
187
204
 
@@ -189,7 +206,6 @@ class UnitPipeline(PipelineTemplate):
189
206
  self.sensors_in_inputs
190
207
  and (not self.output_level_none)
191
208
  and not self.dsms_in_inputs
192
- and not self.depth_maps_in_inputs
193
209
  )
194
210
 
195
211
  if self.output_level_none:
@@ -224,23 +240,15 @@ class UnitPipeline(PipelineTemplate):
224
240
  # Check conf application
225
241
  application_conf = self.check_applications(conf.get(APPLICATIONS, {}))
226
242
 
227
- if (
228
- self.sensors_in_inputs
229
- and not self.depth_maps_in_inputs
230
- and not self.dsms_in_inputs
231
- ):
243
+ if self.sensors_in_inputs and not self.dsms_in_inputs:
232
244
  # Check conf application vs inputs application
233
245
  application_conf = self.check_applications_with_inputs(
234
- self.used_conf[INPUTS], application_conf
246
+ self.used_conf[INPUTS], application_conf, self.res_resamp
235
247
  )
236
248
 
237
249
  self.used_conf[APPLICATIONS] = application_conf
238
250
 
239
- self.config_full_res = copy.deepcopy(self.used_conf)
240
- self.config_full_res.__delitem__("applications")
241
- self.config_full_res[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI] = {}
242
- self.config_full_res[ADVANCED][adv_cst.TERRAIN_A_PRIORI] = {}
243
- self.config_full_res[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI] = True
251
+ self.out_dir = self.used_conf[OUTPUT][out_cst.OUT_DIRECTORY]
244
252
 
245
253
  def quit_on_app(self, app_name):
246
254
  """
@@ -269,34 +277,26 @@ class UnitPipeline(PipelineTemplate):
269
277
  sensor_to_depth_apps = {
270
278
  "grid_generation": 1, # and 5
271
279
  "resampling": 2, # and 8
272
- "hole_detection": 3,
273
- "sparse_matching.sift": 4,
280
+ "sparse_matching": 4,
274
281
  "ground_truth_reprojection": 6,
275
282
  "dense_matching": 8,
276
- "dense_match_filling.1": 9,
277
- "dense_match_filling.2": 10,
283
+ "dense_match_filling": 9,
278
284
  "triangulation": 11,
279
285
  "point_cloud_outlier_removal.1": 12,
280
286
  "point_cloud_outlier_removal.2": 13,
281
287
  }
282
288
 
283
- depth_merge_apps = {
284
- "point_cloud_fusion": 14,
285
- }
286
-
287
289
  depth_to_dsm_apps = {
288
- "pc_denoising": 15,
289
- "point_cloud_rasterization": 16,
290
- "dem_generation": 17,
291
- "dsm_filling.1": 18,
292
- "dsm_filling.2": 19,
293
- "dsm_filling.3": 20,
294
- "auxiliary_filling": 21,
290
+ "point_cloud_rasterization": 15,
291
+ "dem_generation": 16,
292
+ "dsm_filling.1": 17,
293
+ "dsm_filling.2": 18,
294
+ "dsm_filling.3": 19,
295
+ "auxiliary_filling": 20,
295
296
  }
296
297
 
297
298
  self.app_values = {}
298
299
  self.app_values.update(sensor_to_depth_apps)
299
- self.app_values.update(depth_merge_apps)
300
300
  self.app_values.update(depth_to_dsm_apps)
301
301
 
302
302
  app_conf = conf.get(APPLICATIONS, {})
@@ -318,11 +318,7 @@ class UnitPipeline(PipelineTemplate):
318
318
  ).format(key)
319
319
  logging.warning(warn_msg)
320
320
 
321
- elif (
322
- self.sensors_in_inputs
323
- and not self.depth_maps_in_inputs
324
- and not self.dsms_in_inputs
325
- ):
321
+ elif self.sensors_in_inputs and not self.dsms_in_inputs:
326
322
  self.compute_depth_map = True
327
323
  self.last_application_to_run = max(
328
324
  self.last_application_to_run, self.app_values[key]
@@ -330,11 +326,7 @@ class UnitPipeline(PipelineTemplate):
330
326
 
331
327
  elif key in depth_to_dsm_apps:
332
328
 
333
- if not (
334
- self.sensors_in_inputs
335
- or self.depth_maps_in_inputs
336
- or self.dsms_in_inputs
337
- ):
329
+ if not (self.sensors_in_inputs or self.dsms_in_inputs):
338
330
  warn_msg = (
339
331
  "The application {} can only be used when sensor "
340
332
  "images or depth maps are given as an input. "
@@ -343,11 +335,7 @@ class UnitPipeline(PipelineTemplate):
343
335
  logging.warning(warn_msg)
344
336
 
345
337
  else:
346
- if (
347
- self.sensors_in_inputs
348
- and not self.depth_maps_in_inputs
349
- and not self.dsms_in_inputs
350
- ):
338
+ if self.sensors_in_inputs and not self.dsms_in_inputs:
351
339
  self.compute_depth_map = True
352
340
 
353
341
  # enabled to start the depth map to dsm process
@@ -357,43 +345,6 @@ class UnitPipeline(PipelineTemplate):
357
345
  self.last_application_to_run, self.app_values[key]
358
346
  )
359
347
 
360
- elif key in depth_merge_apps:
361
-
362
- if not self.merging:
363
- warn_msg = (
364
- "The application {} can only be used when merging "
365
- "is activated (this parameter is located in the "
366
- "'advanced' config key). "
367
- "The application's configuration will be ignored."
368
- ).format(key)
369
- logging.warning(warn_msg)
370
-
371
- elif not (
372
- self.sensors_in_inputs
373
- or self.depth_maps_in_inputs
374
- or self.dsms_in_inputs
375
- ):
376
- warn_msg = (
377
- "The application {} can only be used when sensor "
378
- "images or depth maps are given as an input. "
379
- "Its configuration will be ignored."
380
- ).format(key)
381
- logging.warning(warn_msg)
382
-
383
- else:
384
- if (
385
- self.sensors_in_inputs
386
- and not self.depth_maps_in_inputs
387
- and not self.dsms_in_inputs
388
- ):
389
- self.compute_depth_map = True
390
-
391
- # enabled to start the depth map to dsm process
392
- self.save_output_point_cloud = True
393
-
394
- self.last_application_to_run = max(
395
- self.last_application_to_run, self.app_values[key]
396
- )
397
348
  else:
398
349
  warn_msg = (
399
350
  "The application {} was not recognized. Its configuration"
@@ -436,45 +387,51 @@ class UnitPipeline(PipelineTemplate):
436
387
  """
437
388
 
438
389
  output_config = {}
439
- if (
440
- sens_cst.SENSORS in conf
441
- and depth_cst.DEPTH_MAPS not in conf
442
- and dsm_cst.DSMS not in conf
443
- ):
390
+ if sens_cst.SENSORS in conf and dsm_cst.DSMS not in conf:
444
391
  output_config = sensor_inputs.sensors_check_inputs(
445
392
  conf, config_dir=config_dir
446
393
  )
447
- elif depth_cst.DEPTH_MAPS in conf:
448
- output_config = {
449
- **output_config,
450
- **depth_map_inputs.check_depth_maps_inputs(
451
- conf, config_dir=config_dir
452
- ),
453
- }
454
- else:
394
+ elif dsm_cst.DSMS in conf:
455
395
  output_config = {
456
396
  **output_config,
457
397
  **dsm_inputs.check_dsm_inputs(conf, config_dir=config_dir),
458
398
  }
399
+ else:
400
+ raise RuntimeError("No sensors or dsms in inputs")
459
401
  return output_config
460
402
 
461
- @staticmethod
462
- def check_output(conf):
403
+ def save_configurations(self):
404
+ """
405
+ Save used_conf and refined_conf configurations
406
+ """
407
+
408
+ cars_dataset.save_dict(
409
+ self.used_conf,
410
+ os.path.join(self.out_dir, "current_res_used_conf.json"),
411
+ safe_save=True,
412
+ )
413
+ cars_dataset.save_dict(
414
+ self.refined_conf,
415
+ os.path.join(self.out_dir, "refined_conf.json"),
416
+ safe_save=True,
417
+ )
418
+
419
+ def check_output(self, conf, scaling_coeff):
463
420
  """
464
421
  Check the output given
465
422
 
466
423
  :param conf: configuration of output
467
424
  :type conf: dict
468
-
469
- :return overloader output
470
- :rtype : dict
425
+ :param scaling_coeff: scaling factor for resolution
426
+ :type scaling_coeff: float
427
+ :return: overloader output
428
+ :rtype: dict
471
429
  """
472
- return output_parameters.check_output_parameters(conf)
430
+ return output_parameters.check_output_parameters(conf, scaling_coeff)
473
431
 
474
432
  def check_applications( # noqa: C901 : too complex
475
433
  self,
476
434
  conf,
477
- key=None,
478
435
  ):
479
436
  """
480
437
  Check the given configuration for applications,
@@ -483,43 +440,18 @@ class UnitPipeline(PipelineTemplate):
483
440
  :param conf: configuration of applications
484
441
  :type conf: dict
485
442
  """
443
+ scaling_coeff = self.scaling_coeff
444
+
445
+ needed_applications = application_parameters.get_needed_apps(
446
+ self.sensors_in_inputs,
447
+ self.save_output_dsm,
448
+ self.save_output_point_cloud,
449
+ conf,
450
+ )
486
451
 
487
452
  # Check if all specified applications are used
488
453
  # Application in terrain_application are note used in
489
454
  # the sensors_to_dense_depth_maps pipeline
490
- needed_applications = []
491
-
492
- if self.sensors_in_inputs:
493
- needed_applications += [
494
- "grid_generation",
495
- "resampling",
496
- "ground_truth_reprojection",
497
- "hole_detection",
498
- "dense_match_filling.1",
499
- "dense_match_filling.2",
500
- "sparse_matching.sift",
501
- "dense_matching",
502
- "triangulation",
503
- "dem_generation",
504
- "point_cloud_outlier_removal.1",
505
- "point_cloud_outlier_removal.2",
506
- ]
507
-
508
- if self.save_output_dsm or self.save_output_point_cloud:
509
- needed_applications += ["pc_denoising"]
510
-
511
- if self.save_output_dsm:
512
- needed_applications += [
513
- "point_cloud_rasterization",
514
- "dsm_filling.1",
515
- "dsm_filling.2",
516
- "dsm_filling.3",
517
- "auxiliary_filling",
518
- ]
519
-
520
- if self.merging: # we have to merge point clouds, add merging apps
521
- needed_applications += ["point_cloud_fusion"]
522
-
523
455
  for app_key in conf.keys():
524
456
  if app_key not in needed_applications:
525
457
  msg = (
@@ -532,28 +464,24 @@ class UnitPipeline(PipelineTemplate):
532
464
  # Initialize used config
533
465
  used_conf = {}
534
466
 
535
- for app_key in [
536
- "point_cloud_outlier_removal.1",
537
- "point_cloud_outlier_removal.2",
538
- "auxiliary_filling",
539
- ]:
540
- if conf.get(app_key) is not None:
541
- config_app = conf.get(app_key)
542
- if "activated" not in config_app:
543
- conf[app_key]["activated"] = True
544
-
545
467
  for app_key in needed_applications:
546
468
  used_conf[app_key] = conf.get(app_key, {})
469
+ if used_conf[app_key] is None:
470
+ continue
547
471
  used_conf[app_key]["save_intermediate_data"] = (
548
472
  self.save_all_intermediate_data
549
473
  or used_conf[app_key].get("save_intermediate_data", False)
550
474
  )
551
475
 
552
- for app_key in [
553
- "point_cloud_fusion",
554
- "pc_denoising",
555
- ]:
556
- if app_key in needed_applications:
476
+ if app_key == "auxiliary_filling":
477
+ if used_conf[app_key] is not None:
478
+ used_conf[app_key]["activated"] = used_conf[app_key].get(
479
+ "activated", True
480
+ )
481
+
482
+ if app_key in [
483
+ "point_cloud_fusion",
484
+ ]:
557
485
  used_conf[app_key]["save_by_pair"] = used_conf[app_key].get(
558
486
  "save_by_pair", self.save_all_point_clouds_by_pair
559
487
  )
@@ -561,26 +489,25 @@ class UnitPipeline(PipelineTemplate):
561
489
  self.epipolar_grid_generation_application = None
562
490
  self.resampling_application = None
563
491
  self.ground_truth_reprojection = None
564
- self.hole_detection_app = None
565
- self.dense_match_filling_1 = None
566
- self.dense_match_filling_2 = None
567
- self.sparse_mtch_sift_app = None
492
+ self.dense_match_filling = None
493
+ self.sparse_mtch_app = None
568
494
  self.dense_matching_app = None
569
495
  self.triangulation_application = None
570
496
  self.dem_generation_application = None
571
- self.pc_denoising_application = None
572
- self.pc_outlier_removal_1_app = None
573
- self.pc_outlier_removal_2_app = None
497
+ self.pc_outlier_removal_apps = {}
574
498
  self.rasterization_application = None
575
499
  self.pc_fusion_application = None
576
500
  self.dsm_filling_1_application = None
577
501
  self.dsm_filling_2_application = None
578
502
  self.dsm_filling_3_application = None
503
+ self.dsm_filling_apps = {}
579
504
 
580
505
  if self.sensors_in_inputs:
581
506
  # Epipolar grid generation
582
507
  self.epipolar_grid_generation_application = Application(
583
- "grid_generation", cfg=used_conf.get("grid_generation", {})
508
+ "grid_generation",
509
+ cfg=used_conf.get("grid_generation", {}),
510
+ scaling_coeff=scaling_coeff,
584
511
  )
585
512
  used_conf["grid_generation"] = (
586
513
  self.epipolar_grid_generation_application.get_conf()
@@ -588,7 +515,9 @@ class UnitPipeline(PipelineTemplate):
588
515
 
589
516
  # image resampling
590
517
  self.resampling_application = Application(
591
- "resampling", cfg=used_conf.get("resampling", {})
518
+ "resampling",
519
+ cfg=used_conf.get("resampling", {}),
520
+ scaling_coeff=scaling_coeff,
592
521
  )
593
522
  used_conf["resampling"] = self.resampling_application.get_conf()
594
523
 
@@ -610,45 +539,29 @@ class UnitPipeline(PipelineTemplate):
610
539
  self.ground_truth_reprojection = Application(
611
540
  "ground_truth_reprojection",
612
541
  cfg=used_conf.get("ground_truth_reprojection", {}),
542
+ scaling_coeff=scaling_coeff,
613
543
  )
614
- # holes detection
615
- self.hole_detection_app = Application(
616
- "hole_detection", cfg=used_conf.get("hole_detection", {})
617
- )
618
- used_conf["hole_detection"] = self.hole_detection_app.get_conf()
619
-
620
- # disparity filling 1 plane
621
- self.dense_match_filling_1 = Application(
622
- "dense_match_filling",
623
- cfg=used_conf.get(
624
- "dense_match_filling.1",
625
- {"method": "plane"},
626
- ),
627
- )
628
- used_conf["dense_match_filling.1"] = (
629
- self.dense_match_filling_1.get_conf()
630
- )
631
544
 
632
- # disparity filling 2
633
- self.dense_match_filling_2 = Application(
545
+ # disparity filling
546
+ self.dense_match_filling = Application(
634
547
  "dense_match_filling",
635
548
  cfg=used_conf.get(
636
- "dense_match_filling.2",
549
+ "dense_match_filling",
637
550
  {"method": "zero_padding"},
638
551
  ),
552
+ scaling_coeff=scaling_coeff,
639
553
  )
640
- used_conf["dense_match_filling.2"] = (
641
- self.dense_match_filling_2.get_conf()
554
+ used_conf["dense_match_filling"] = (
555
+ self.dense_match_filling.get_conf()
642
556
  )
643
557
 
644
558
  # Sparse Matching
645
- self.sparse_mtch_sift_app = Application(
559
+ self.sparse_mtch_app = Application(
646
560
  "sparse_matching",
647
- cfg=used_conf.get("sparse_matching.sift", {"method": "sift"}),
648
- )
649
- used_conf["sparse_matching.sift"] = (
650
- self.sparse_mtch_sift_app.get_conf()
561
+ cfg=used_conf.get("sparse_matching", {"method": "sift"}),
562
+ scaling_coeff=scaling_coeff,
651
563
  )
564
+ used_conf["sparse_matching"] = self.sparse_mtch_app.get_conf()
652
565
 
653
566
  # Matching
654
567
  generate_performance_map = (
@@ -671,14 +584,26 @@ class UnitPipeline(PipelineTemplate):
671
584
  is None
672
585
  ):
673
586
  dense_matching_config["performance_map_method"] = "risk"
587
+
588
+ # particular case for some epipolar resolutions
589
+ if not dense_matching_config:
590
+ used_conf["dense_matching"]["performance_map_method"] = [
591
+ "risk",
592
+ "intervals",
593
+ ]
594
+
674
595
  self.dense_matching_app = Application(
675
- "dense_matching", cfg=dense_matching_config
596
+ "dense_matching",
597
+ cfg=dense_matching_config,
598
+ scaling_coeff=scaling_coeff,
676
599
  )
677
600
  used_conf["dense_matching"] = self.dense_matching_app.get_conf()
678
601
 
679
602
  # Triangulation
680
603
  self.triangulation_application = Application(
681
- "triangulation", cfg=used_conf.get("triangulation", {})
604
+ "triangulation",
605
+ cfg=used_conf.get("triangulation", {}),
606
+ scaling_coeff=scaling_coeff,
682
607
  )
683
608
  used_conf["triangulation"] = (
684
609
  self.triangulation_application.get_conf()
@@ -686,123 +611,134 @@ class UnitPipeline(PipelineTemplate):
686
611
 
687
612
  # MNT generation
688
613
  self.dem_generation_application = Application(
689
- "dem_generation", cfg=used_conf.get("dem_generation", {})
614
+ "dem_generation",
615
+ cfg=used_conf.get("dem_generation", {}),
616
+ scaling_coeff=scaling_coeff,
690
617
  )
691
618
  used_conf["dem_generation"] = (
692
619
  self.dem_generation_application.get_conf()
693
620
  )
694
621
 
695
- # Points cloud small component outlier removal
696
- if "point_cloud_outlier_removal.1" in used_conf:
697
- if "method" not in used_conf["point_cloud_outlier_removal.1"]:
698
- used_conf["point_cloud_outlier_removal.1"][
699
- "method"
700
- ] = "small_components"
701
- self.pc_outlier_removal_1_app = Application(
702
- "point_cloud_outlier_removal",
703
- cfg=used_conf.get(
704
- "point_cloud_outlier_removal.1",
705
- {"method": "small_components"},
706
- ),
707
- )
708
- used_conf["point_cloud_outlier_removal.1"] = (
709
- self.pc_outlier_removal_1_app.get_conf()
710
- )
622
+ for app_key, app_conf in used_conf.items():
623
+ if not app_key.startswith("point_cloud_outlier_removal"):
624
+ continue
711
625
 
712
- # Points cloud statistical outlier removal
713
- self.pc_outlier_removal_2_app = Application(
714
- "point_cloud_outlier_removal",
715
- cfg=used_conf.get(
716
- "point_cloud_outlier_removal.2",
717
- {"method": "statistical"},
718
- ),
626
+ if app_conf is None:
627
+ self.pc_outlier_removal_apps = {}
628
+ # keep over multiple runs
629
+ used_conf["point_cloud_outlier_removal"] = None
630
+ break
631
+
632
+ if app_key in self.pc_outlier_removal_apps:
633
+ msg = (
634
+ f"The key {app_key} is defined twice in the input "
635
+ "configuration."
636
+ )
637
+ logging.error(msg)
638
+ raise NameError(msg)
639
+
640
+ if app_key[27:] == ".1":
641
+ app_conf.setdefault("method", "small_components")
642
+ if app_key[27:] == ".2":
643
+ app_conf.setdefault("method", "statistical")
644
+
645
+ self.pc_outlier_removal_apps[app_key] = Application(
646
+ "point_cloud_outlier_removal",
647
+ cfg=app_conf,
648
+ scaling_coeff=scaling_coeff,
649
+ )
650
+ used_conf[app_key] = self.pc_outlier_removal_apps[
651
+ app_key
652
+ ].get_conf()
653
+
654
+ methods_str = "\n".join(
655
+ f" - {k}={a.used_method}"
656
+ for k, a in self.pc_outlier_removal_apps.items()
719
657
  )
720
- used_conf["point_cloud_outlier_removal.2"] = (
721
- self.pc_outlier_removal_2_app.get_conf()
658
+ logging.info(
659
+ "{} point cloud outlier removal apps registered:\n{}".format(
660
+ len(self.pc_outlier_removal_apps), methods_str
661
+ )
722
662
  )
723
663
 
724
664
  if self.save_output_dsm or self.save_output_point_cloud:
725
665
 
726
- # Point cloud denoising
727
- self.pc_denoising_application = Application(
728
- "pc_denoising",
729
- cfg=used_conf.get("pc_denoising", {"method": "none"}),
730
- )
731
- used_conf["pc_denoising"] = self.pc_denoising_application.get_conf()
732
-
733
666
  if self.save_output_dsm:
734
667
 
735
668
  # Rasterization
736
669
  self.rasterization_application = Application(
737
670
  "point_cloud_rasterization",
738
671
  cfg=used_conf.get("point_cloud_rasterization", {}),
672
+ scaling_coeff=scaling_coeff,
739
673
  )
740
674
  used_conf["point_cloud_rasterization"] = (
741
675
  self.rasterization_application.get_conf()
742
676
  )
743
- # DSM filling 1 : Exogenous filling
744
- self.dsm_filling_1_application = Application(
745
- "dsm_filling",
746
- cfg=conf.get(
747
- "dsm_filling.1",
748
- {"method": "exogenous_filling"},
749
- ),
750
- )
751
- used_conf["dsm_filling.1"] = (
752
- self.dsm_filling_1_application.get_conf()
753
- )
754
- # DSM filling 2 : Bulldozer
755
- self.dsm_filling_2_application = Application(
756
- "dsm_filling",
757
- cfg=conf.get(
758
- "dsm_filling.2",
759
- {"method": "bulldozer"},
760
- ),
761
- )
762
- used_conf["dsm_filling.2"] = (
763
- self.dsm_filling_2_application.get_conf()
764
- )
765
- # DSM filling 3 : Border interpolation
766
- self.dsm_filling_3_application = Application(
767
- "dsm_filling",
768
- cfg=conf.get(
769
- "dsm_filling.3",
770
- {"method": "border_interpolation"},
771
- ),
677
+
678
+ for app_key, app_conf in used_conf.items():
679
+ if not app_key.startswith("dsm_filling"):
680
+ continue
681
+
682
+ if app_conf is None:
683
+ self.dsm_filling_apps = {}
684
+ # keep over multiple runs
685
+ used_conf["dsm_filling"] = None
686
+ break
687
+
688
+ if app_key in self.dsm_filling_apps:
689
+ msg = (
690
+ f"The key {app_key} is defined twice in the input "
691
+ "configuration."
692
+ )
693
+ logging.error(msg)
694
+ raise NameError(msg)
695
+
696
+ if app_key[11:] == ".1":
697
+ app_conf.setdefault("method", "exogenous_filling")
698
+ if app_key[11:] == ".2":
699
+ app_conf.setdefault("method", "bulldozer")
700
+ if app_key[11:] == ".3":
701
+ app_conf.setdefault("method", "border_interpolation")
702
+
703
+ self.dsm_filling_apps[app_key] = Application(
704
+ "dsm_filling",
705
+ cfg=app_conf,
706
+ scaling_coeff=scaling_coeff,
707
+ )
708
+ used_conf[app_key] = self.dsm_filling_apps[
709
+ app_key
710
+ ].get_conf()
711
+
712
+ methods_str = "\n".join(
713
+ f" - {k}={a.used_method}"
714
+ for k, a in self.dsm_filling_apps.items()
772
715
  )
773
- used_conf["dsm_filling.3"] = (
774
- self.dsm_filling_3_application.get_conf()
716
+ logging.info(
717
+ "{} dsm filling apps registered:\n{}".format(
718
+ len(self.dsm_filling_apps), methods_str
719
+ )
775
720
  )
721
+
776
722
  # Auxiliary filling
777
723
  self.auxiliary_filling_application = Application(
778
- "auxiliary_filling", cfg=conf.get("auxiliary_filling", {})
724
+ "auxiliary_filling",
725
+ cfg=conf.get("auxiliary_filling", {}),
726
+ scaling_coeff=scaling_coeff,
779
727
  )
780
728
  used_conf["auxiliary_filling"] = (
781
729
  self.auxiliary_filling_application.get_conf()
782
730
  )
783
731
 
784
- if (
785
- self.dsm_filling_1_application.classification
786
- or self.dsm_filling_2_application.classification
787
- or self.dsm_filling_3_application.classification
732
+ if any(
733
+ app_obj.classification != ["nodata"]
734
+ for app_key, app_obj in self.dsm_filling_apps.items()
788
735
  ):
789
736
  self.save_output_classif_for_filling = True
790
737
 
791
- if self.merging:
792
-
793
- # Point cloud fusion
794
- self.pc_fusion_application = Application(
795
- "point_cloud_fusion",
796
- cfg=used_conf.get("point_cloud_fusion", {}),
797
- )
798
- used_conf["point_cloud_fusion"] = (
799
- self.pc_fusion_application.get_conf()
800
- )
801
-
802
738
  return used_conf
803
739
 
804
740
  def check_applications_with_inputs( # noqa: C901 : too complex
805
- self, inputs_conf, application_conf
741
+ self, inputs_conf, application_conf, epipolar_resolution
806
742
  ):
807
743
  """
808
744
  Check for each application the input and output configuration
@@ -812,55 +748,34 @@ class UnitPipeline(PipelineTemplate):
812
748
  :type inputs_conf: dict
813
749
  :param application_conf: application checked configuration
814
750
  :type application_conf: dict
751
+ :param epipolar_resolution: epipolar resolution
752
+ :type epipolar_resolution: int
815
753
  """
816
754
 
817
755
  initial_elevation = (
818
756
  inputs_conf[sens_cst.INITIAL_ELEVATION]["dem"] is not None
819
757
  )
820
- if self.sparse_mtch_sift_app.elevation_delta_lower_bound is None:
821
- self.sparse_mtch_sift_app.used_config[
822
- "elevation_delta_lower_bound"
823
- ] = (-500 if initial_elevation else -1000)
824
- self.sparse_mtch_sift_app.elevation_delta_lower_bound = (
825
- self.sparse_mtch_sift_app.used_config[
826
- "elevation_delta_lower_bound"
827
- ]
758
+ if self.sparse_mtch_app.elevation_delta_lower_bound is None:
759
+ self.sparse_mtch_app.used_config["elevation_delta_lower_bound"] = (
760
+ -500 if initial_elevation else -1000
828
761
  )
829
- if self.sparse_mtch_sift_app.elevation_delta_upper_bound is None:
830
- self.sparse_mtch_sift_app.used_config[
831
- "elevation_delta_upper_bound"
832
- ] = (1000 if initial_elevation else 9000)
833
- self.sparse_mtch_sift_app.elevation_delta_upper_bound = (
834
- self.sparse_mtch_sift_app.used_config[
835
- "elevation_delta_upper_bound"
836
- ]
762
+ self.sparse_mtch_app.elevation_delta_lower_bound = (
763
+ self.sparse_mtch_app.used_config["elevation_delta_lower_bound"]
837
764
  )
838
- application_conf["sparse_matching.sift"] = (
839
- self.sparse_mtch_sift_app.get_conf()
840
- )
841
-
842
- if (
843
- application_conf["dem_generation"]["method"]
844
- == "bulldozer_on_raster"
845
- ):
846
- first_image_path = next(iter(inputs_conf["sensors"].values()))[
847
- "image"
848
- ]["main_file"]
849
- first_image_size = rasterio_get_size(first_image_path)
850
- first_image_nb_pixels = math.prod(first_image_size)
851
- dem_gen_used_mem = first_image_nb_pixels / 1e8
852
- if dem_gen_used_mem > 8:
853
- logging.warning(
854
- "DEM generation method is 'bulldozer_on_raster'. "
855
- f"This method can use up to {dem_gen_used_mem} Gb "
856
- "of memory. If you think that it is too much for "
857
- "your computer, you can re-lauch the run using "
858
- "'dichotomic' method for DEM generation"
859
- )
765
+ if self.sparse_mtch_app.elevation_delta_upper_bound is None:
766
+ self.sparse_mtch_app.used_config["elevation_delta_upper_bound"] = (
767
+ 1000 if initial_elevation else 9000
768
+ )
769
+ self.sparse_mtch_app.elevation_delta_upper_bound = (
770
+ self.sparse_mtch_app.used_config["elevation_delta_upper_bound"]
771
+ )
772
+ application_conf["sparse_matching"] = self.sparse_mtch_app.get_conf()
860
773
 
861
774
  # check classification application parameter compare
862
775
  # to each sensors inputs classification list
863
776
  for application_key in application_conf:
777
+ if application_conf[application_key] is None:
778
+ continue
864
779
  if "classification" in application_conf[application_key]:
865
780
  for item in inputs_conf["sensors"]:
866
781
  if "classification" in inputs_conf["sensors"][item].keys():
@@ -895,44 +810,123 @@ class UnitPipeline(PipelineTemplate):
895
810
  )
896
811
  for key1, key2 in inputs_conf["pairing"]:
897
812
  corr_cfg = self.dense_matching_app.loader.get_conf()
898
- img_left = inputs_conf["sensors"][key1]["image"]["main_file"]
899
- img_right = inputs_conf["sensors"][key2]["image"]["main_file"]
813
+ nodata_left = inputs_conf["sensors"][key2]["image"]["no_data"]
814
+ nodata_right = inputs_conf["sensors"][key2]["image"]["no_data"]
900
815
  bands_left = list(
901
816
  inputs_conf["sensors"][key1]["image"]["bands"].keys()
902
817
  )
903
818
  bands_right = list(
904
819
  inputs_conf["sensors"][key2]["image"]["bands"].keys()
905
820
  )
906
- classif_left = None
907
- classif_right = None
821
+ bands_classif_left = None
822
+ bands_classif_right = None
908
823
  if (
909
824
  "classification" in inputs_conf["sensors"][key1]
910
825
  and inputs_conf["sensors"][key1]["classification"] is not None
911
826
  ):
912
- classif_left = inputs_conf["sensors"][key1]["classification"][
913
- "main_file"
914
- ]
827
+ bands_classif_left = inputs_conf["sensors"][key1][
828
+ "classification"
829
+ ]["bands"].keys()
915
830
  if (
916
831
  "classification" in inputs_conf["sensors"][key2]
917
- and inputs_conf["sensors"][key1]["classification"] is not None
832
+ and inputs_conf["sensors"][key2]["classification"] is not None
918
833
  ):
919
- classif_right = inputs_conf["sensors"][key2]["classification"][
920
- "main_file"
921
- ]
834
+ bands_classif_right = inputs_conf["sensors"][key2][
835
+ "classification"
836
+ ]["bands"].keys()
922
837
  self.dense_matching_app.corr_config = (
923
838
  self.dense_matching_app.loader.check_conf(
924
839
  corr_cfg,
925
- img_left,
926
- img_right,
840
+ nodata_left,
841
+ nodata_right,
927
842
  bands_left,
928
843
  bands_right,
929
- classif_left,
930
- classif_right,
844
+ bands_classif_left,
845
+ bands_classif_right,
931
846
  )
932
847
  )
933
848
 
849
+ # Change the step regarding the resolution
850
+ # For the small resolution, the resampling perform better
851
+ # with a small step
852
+ # For the higher ones, a step at 30 should be better
853
+ first_image_path = next(iter(inputs_conf["sensors"].values()))["image"][
854
+ "main_file"
855
+ ]
856
+ first_image_size = rasterio_get_size(first_image_path)
857
+ size_low_res_img_row = first_image_size[0] // epipolar_resolution
858
+ size_low_res_img_col = first_image_size[1] // epipolar_resolution
859
+ if epipolar_resolution > 1:
860
+ if size_low_res_img_row <= 900 and size_low_res_img_col <= 900:
861
+ application_conf["grid_generation"]["epi_step"] = (
862
+ epipolar_resolution * 5
863
+ )
864
+ else:
865
+ application_conf["grid_generation"]["epi_step"] = (
866
+ epipolar_resolution * 30
867
+ )
868
+
934
869
  return application_conf
935
870
 
871
+ def generate_grid_correction_on_dem(self, pair_key, geo_plugin_on_dem):
872
+ """
873
+ Generate the epipolar grid correction for a given pair, using given dem
874
+ """
875
+
876
+ # Generate new grids with dem
877
+ # Generate rectification grids
878
+ (
879
+ grid_left_new_dem,
880
+ grid_right_new_dem,
881
+ ) = self.epipolar_grid_generation_application.run(
882
+ self.pairs[pair_key]["sensor_image_left"],
883
+ self.pairs[pair_key]["sensor_image_right"],
884
+ geo_plugin_on_dem,
885
+ orchestrator=self.cars_orchestrator,
886
+ pair_folder=os.path.join(
887
+ self.dump_dir,
888
+ "epipolar_grid_generation",
889
+ "new_dem",
890
+ pair_key,
891
+ ),
892
+ pair_key=pair_key,
893
+ )
894
+
895
+ if self.pairs[pair_key].get("sensor_matches_left", None) is None:
896
+ logging.error(
897
+ "No sensor matches available to compute grid correction"
898
+ )
899
+ return None
900
+
901
+ # Generate new matches with new grids
902
+ new_grid_matches_array = geo_plugin_on_dem.transform_matches_from_grids(
903
+ self.pairs[pair_key]["sensor_matches_left"],
904
+ self.pairs[pair_key]["sensor_matches_right"],
905
+ grid_left_new_dem,
906
+ grid_right_new_dem,
907
+ )
908
+
909
+ # Generate grid_correction
910
+ # Compute grid correction
911
+ (
912
+ new_grid_correction_coef,
913
+ _,
914
+ _,
915
+ _,
916
+ ) = grid_correction_app.estimate_right_grid_correction(
917
+ new_grid_matches_array,
918
+ grid_right_new_dem,
919
+ save_matches=False,
920
+ minimum_nb_matches=0,
921
+ pair_folder=os.path.join(
922
+ self.dump_dir, "grid_correction", " new_dem", pair_key
923
+ ),
924
+ pair_key=pair_key,
925
+ orchestrator=self.cars_orchestrator,
926
+ )
927
+
928
+ return new_grid_correction_coef
929
+
936
930
  def sensor_to_depth_maps(self): # noqa: C901
937
931
  """
938
932
  Creates the depth map from the sensor images given in the input,
@@ -956,7 +950,7 @@ class UnitPipeline(PipelineTemplate):
956
950
  self.input_roi_poly, self.input_roi_epsg, self.epsg
957
951
  )
958
952
 
959
- self.resolution = output[out_cst.RESOLUTION] * self.res_resamp
953
+ self.resolution = output[out_cst.RESOLUTION]
960
954
 
961
955
  # List of terrain roi corresponding to each epipolar pair
962
956
  # Used to generate final terrain roi
@@ -991,7 +985,7 @@ class UnitPipeline(PipelineTemplate):
991
985
  # used in dem generation
992
986
  self.triangulated_matches_list = []
993
987
 
994
- save_matches = self.sparse_mtch_sift_app.get_save_matches()
988
+ save_matches = self.sparse_mtch_app.get_save_matches()
995
989
 
996
990
  save_corrected_grid = (
997
991
  self.epipolar_grid_generation_application.get_save_grids()
@@ -1014,21 +1008,10 @@ class UnitPipeline(PipelineTemplate):
1014
1008
  # We generate grids with dem if it is provided.
1015
1009
  # If not provided, grid are generated without dem and a dem
1016
1010
  # will be generated, to use later for a new grid generation**
1017
- altitude_delta_min = inputs.get(sens_cst.INITIAL_ELEVATION, {}).get(
1018
- sens_cst.ALTITUDE_DELTA_MIN, None
1019
- )
1020
- altitude_delta_max = inputs.get(sens_cst.INITIAL_ELEVATION, {}).get(
1021
- sens_cst.ALTITUDE_DELTA_MAX, None
1022
- )
1023
1011
 
1024
1012
  if inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] is None:
1025
1013
  geom_plugin = self.geom_plugin_without_dem_and_geoid
1026
1014
 
1027
- if None not in (altitude_delta_min, altitude_delta_max):
1028
- raise RuntimeError(
1029
- "Dem path is mandatory for "
1030
- "the use of altitude deltas"
1031
- )
1032
1015
  else:
1033
1016
  geom_plugin = self.geom_plugin_with_dem_and_geoid
1034
1017
 
@@ -1053,43 +1036,15 @@ class UnitPipeline(PipelineTemplate):
1053
1036
  if self.quit_on_app("grid_generation"):
1054
1037
  continue # keep iterating over pairs, but don't go further
1055
1038
 
1056
- # Run holes detection
1057
- # Get classif depending on which filling is used
1058
- # For now, 2 filling application can be used, and be configured
1059
- # with any order. the .1 will be performed before the .2
1060
- self.pairs[pair_key]["holes_classif"] = []
1061
- self.pairs[pair_key]["holes_poly_margin"] = 0
1062
- add_classif = False
1063
- if self.dense_match_filling_1.used_method == "plane":
1064
- self.pairs[pair_key][
1065
- "holes_classif"
1066
- ] += self.dense_match_filling_1.get_classif()
1067
- self.pairs[pair_key]["holes_poly_margin"] = max(
1068
- self.pairs[pair_key]["holes_poly_margin"],
1069
- self.dense_match_filling_1.get_poly_margin(),
1070
- )
1071
- add_classif = True
1072
- if self.dense_match_filling_2.used_method == "plane":
1073
- self.pairs[pair_key][
1074
- "holes_classif"
1075
- ] += self.dense_match_filling_2.get_classif()
1076
- self.pairs[pair_key]["holes_poly_margin"] = max(
1077
- self.pairs[pair_key]["holes_poly_margin"],
1078
- self.dense_match_filling_2.get_poly_margin(),
1079
- )
1080
- add_classif = True
1081
-
1082
- self.pairs[pair_key]["holes_bbox_left"] = []
1083
- self.pairs[pair_key]["holes_bbox_right"] = []
1084
-
1085
- if self.used_conf[ADVANCED][
1086
- adv_cst.USE_EPIPOLAR_A_PRIORI
1087
- ] is False or (len(self.pairs[pair_key]["holes_classif"]) > 0):
1039
+ if self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] in (
1040
+ None,
1041
+ {},
1042
+ ):
1088
1043
  # Run resampling only if needed:
1089
- # no a priori or needs to detect holes
1044
+ # no a priori
1090
1045
 
1091
1046
  # Get required bands of first resampling
1092
- required_bands = self.sparse_mtch_sift_app.get_required_bands()
1047
+ required_bands = self.sparse_mtch_app.get_required_bands()
1093
1048
 
1094
1049
  # Run first epipolar resampling
1095
1050
  (
@@ -1106,49 +1061,30 @@ class UnitPipeline(PipelineTemplate):
1106
1061
  self.dump_dir, "resampling", "initial", pair_key
1107
1062
  ),
1108
1063
  pair_key=pair_key,
1109
- margins_fun=self.sparse_mtch_sift_app.get_margins_fun(),
1064
+ margins_fun=self.sparse_mtch_app.get_margins_fun(),
1110
1065
  tile_width=None,
1111
1066
  tile_height=None,
1112
- add_classif=add_classif,
1113
1067
  required_bands=required_bands,
1114
1068
  )
1115
1069
 
1116
1070
  if self.quit_on_app("resampling"):
1117
1071
  continue # keep iterating over pairs, but don't go further
1118
1072
 
1119
- # Generate the holes polygons in epipolar images
1120
- # They are only generated if dense_match_filling
1121
- # applications are used later
1122
- (
1123
- self.pairs[pair_key]["holes_bbox_left"],
1124
- self.pairs[pair_key]["holes_bbox_right"],
1125
- ) = self.hole_detection_app.run(
1126
- self.pairs[pair_key]["epipolar_image_left"],
1127
- self.pairs[pair_key]["epipolar_image_right"],
1128
- classification=self.pairs[pair_key]["holes_classif"],
1129
- margin=self.pairs[pair_key]["holes_poly_margin"],
1130
- orchestrator=self.cars_orchestrator,
1131
- pair_folder=os.path.join(
1132
- self.dump_dir, "hole_detection", pair_key
1133
- ),
1134
- pair_key=pair_key,
1135
- )
1136
-
1137
- if self.quit_on_app("hole_detection"):
1138
- continue # keep iterating over pairs, but don't go further
1139
-
1140
- if self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI] is False:
1073
+ if self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] in (
1074
+ None,
1075
+ {},
1076
+ ):
1141
1077
  # Run epipolar sparse_matching application
1142
1078
  (
1143
1079
  self.pairs[pair_key]["epipolar_matches_left"],
1144
1080
  _,
1145
- ) = self.sparse_mtch_sift_app.run(
1081
+ ) = self.sparse_mtch_app.run(
1146
1082
  self.pairs[pair_key]["epipolar_image_left"],
1147
1083
  self.pairs[pair_key]["epipolar_image_right"],
1148
1084
  self.pairs[pair_key]["grid_left"]["disp_to_alt_ratio"],
1149
1085
  orchestrator=self.cars_orchestrator,
1150
1086
  pair_folder=os.path.join(
1151
- self.dump_dir, "sparse_matching.sift", pair_key
1087
+ self.dump_dir, "sparse_matching", pair_key
1152
1088
  ),
1153
1089
  pair_key=pair_key,
1154
1090
  )
@@ -1156,12 +1092,14 @@ class UnitPipeline(PipelineTemplate):
1156
1092
  # Run cluster breakpoint to compute sifts: force computation
1157
1093
  self.cars_orchestrator.breakpoint()
1158
1094
 
1095
+ minimum_nb_matches = self.sparse_mtch_app.get_minimum_nb_matches()
1096
+
1159
1097
  # Run grid correction application
1160
- if self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI] is False:
1098
+ if self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] in (None, {}):
1161
1099
  # Estimate grid correction if no epipolar a priori
1162
1100
  # Filter and save matches
1163
1101
  self.pairs[pair_key]["matches_array"] = (
1164
- self.sparse_mtch_sift_app.filter_matches(
1102
+ self.sparse_mtch_app.filter_matches(
1165
1103
  self.pairs[pair_key]["epipolar_matches_left"],
1166
1104
  self.pairs[pair_key]["grid_left"],
1167
1105
  self.pairs[pair_key]["grid_right"],
@@ -1169,18 +1107,12 @@ class UnitPipeline(PipelineTemplate):
1169
1107
  orchestrator=self.cars_orchestrator,
1170
1108
  pair_key=pair_key,
1171
1109
  pair_folder=os.path.join(
1172
- self.dump_dir, "sparse_matching.sift", pair_key
1173
- ),
1174
- save_matches=(
1175
- self.sparse_mtch_sift_app.get_save_matches()
1110
+ self.dump_dir, "sparse_matching", pair_key
1176
1111
  ),
1112
+ save_matches=(self.sparse_mtch_app.get_save_matches()),
1177
1113
  )
1178
1114
  )
1179
1115
 
1180
- minimum_nb_matches = (
1181
- self.sparse_mtch_sift_app.get_minimum_nb_matches()
1182
- )
1183
-
1184
1116
  # Compute grid correction
1185
1117
  (
1186
1118
  self.pairs[pair_key]["grid_correction_coef"],
@@ -1217,7 +1149,7 @@ class UnitPipeline(PipelineTemplate):
1217
1149
  pair_key
1218
1150
  ]["grid_left"]
1219
1151
 
1220
- if self.quit_on_app("sparse_matching.sift"):
1152
+ if self.quit_on_app("sparse_matching"):
1221
1153
  continue
1222
1154
 
1223
1155
  # Shrink disparity intervals according to SIFT disparities
@@ -1225,7 +1157,7 @@ class UnitPipeline(PipelineTemplate):
1225
1157
  "disp_to_alt_ratio"
1226
1158
  ]
1227
1159
  disp_bounds_params = (
1228
- self.sparse_mtch_sift_app.disparity_bounds_estimation
1160
+ self.sparse_mtch_app.disparity_bounds_estimation
1229
1161
  )
1230
1162
 
1231
1163
  if disp_bounds_params["activated"]:
@@ -1253,11 +1185,11 @@ class UnitPipeline(PipelineTemplate):
1253
1185
  )
1254
1186
  else:
1255
1187
  disp_min = (
1256
- -self.sparse_mtch_sift_app.elevation_delta_upper_bound
1188
+ -self.sparse_mtch_app.elevation_delta_upper_bound
1257
1189
  / disp_to_alt_ratio
1258
1190
  )
1259
1191
  disp_max = (
1260
- -self.sparse_mtch_sift_app.elevation_delta_lower_bound
1192
+ -self.sparse_mtch_app.elevation_delta_lower_bound
1261
1193
  / disp_to_alt_ratio
1262
1194
  )
1263
1195
  logging.info(
@@ -1300,12 +1232,11 @@ class UnitPipeline(PipelineTemplate):
1300
1232
  if (
1301
1233
  self.quit_on_app("grid_generation")
1302
1234
  or self.quit_on_app("resampling")
1303
- or self.quit_on_app("hole_detection")
1304
- or self.quit_on_app("sparse_matching.sift")
1235
+ or self.quit_on_app("sparse_matching")
1305
1236
  ):
1306
1237
  return True
1307
1238
 
1308
- if self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI]:
1239
+ if not self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] in (None, {}):
1309
1240
  # Use a priori
1310
1241
  dem_median = self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI][
1311
1242
  adv_cst.DEM_MEDIAN
@@ -1316,50 +1247,10 @@ class UnitPipeline(PipelineTemplate):
1316
1247
  dem_max = self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI][
1317
1248
  adv_cst.DEM_MAX
1318
1249
  ]
1319
- altitude_delta_min = self.used_conf[ADVANCED][
1320
- adv_cst.TERRAIN_A_PRIORI
1321
- ][adv_cst.ALTITUDE_DELTA_MIN]
1322
- altitude_delta_max = self.used_conf[ADVANCED][
1323
- adv_cst.TERRAIN_A_PRIORI
1324
- ][adv_cst.ALTITUDE_DELTA_MAX]
1325
-
1326
- # update used configuration with terrain a priori
1327
- if None not in (altitude_delta_min, altitude_delta_max):
1328
- advanced_parameters.update_conf(
1329
- self.used_conf,
1330
- dem_median=dem_median,
1331
- altitude_delta_min=altitude_delta_min,
1332
- altitude_delta_max=altitude_delta_max,
1333
- )
1334
- else:
1335
- advanced_parameters.update_conf(
1336
- self.used_conf,
1337
- dem_median=dem_median,
1338
- dem_min=dem_min,
1339
- dem_max=dem_max,
1340
- )
1341
-
1342
- advanced_parameters.update_conf(
1343
- self.config_full_res,
1344
- dem_median=dem_median,
1345
- dem_min=dem_min,
1346
- dem_max=dem_max,
1347
- )
1348
-
1349
- # quit only after the configuration was updated
1350
- if self.quit_on_app("dem_generation"):
1351
- return True
1352
1250
 
1353
1251
  # Define param
1354
1252
  use_global_disp_range = self.dense_matching_app.use_global_disp_range
1355
1253
 
1356
- if self.pc_denoising_application is not None:
1357
- denoising_overload_fun = (
1358
- self.pc_denoising_application.get_triangulation_overload()
1359
- )
1360
- else:
1361
- denoising_overload_fun = None
1362
-
1363
1254
  self.pairs_names = [
1364
1255
  pair_name for pair_name, _, _ in self.list_sensor_pairs
1365
1256
  ]
@@ -1368,7 +1259,7 @@ class UnitPipeline(PipelineTemplate):
1368
1259
  # Geometry plugin with dem will be used for the grid generation
1369
1260
  geom_plugin = self.geom_plugin_with_dem_and_geoid
1370
1261
 
1371
- if self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI] is False:
1262
+ if self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] in (None, {}):
1372
1263
  save_matches = True
1373
1264
 
1374
1265
  (
@@ -1384,7 +1275,8 @@ class UnitPipeline(PipelineTemplate):
1384
1275
  save_matches=save_matches,
1385
1276
  )
1386
1277
  elif (
1387
- self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI] is True
1278
+ not self.used_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI]
1279
+ in (None, {})
1388
1280
  and not self.use_sift_a_priori
1389
1281
  ):
1390
1282
  # Use epipolar a priori
@@ -1395,7 +1287,7 @@ class UnitPipeline(PipelineTemplate):
1395
1287
  ][pair_key][adv_cst.DISPARITY_RANGE]
1396
1288
 
1397
1289
  advanced_parameters.update_conf(
1398
- self.config_full_res,
1290
+ self.refined_conf,
1399
1291
  dmin=dmin,
1400
1292
  dmax=dmax,
1401
1293
  pair_key=pair_key,
@@ -1434,7 +1326,7 @@ class UnitPipeline(PipelineTemplate):
1434
1326
  # Correct grids with former matches
1435
1327
  # Transform matches to new grids
1436
1328
 
1437
- save_matches = self.sparse_mtch_sift_app.get_save_matches()
1329
+ save_matches = self.sparse_mtch_app.get_save_matches()
1438
1330
 
1439
1331
  self.sensor_matches_left = os.path.join(
1440
1332
  self.first_res_out_dir,
@@ -1475,6 +1367,7 @@ class UnitPipeline(PipelineTemplate):
1475
1367
  new_grid_matches_array,
1476
1368
  self.pairs[pair_key]["grid_right"],
1477
1369
  save_matches=save_matches,
1370
+ minimum_nb_matches=minimum_nb_matches,
1478
1371
  pair_folder=os.path.join(
1479
1372
  self.dump_dir, "grid_correction", "new", pair_key
1480
1373
  ),
@@ -1515,28 +1408,19 @@ class UnitPipeline(PipelineTemplate):
1515
1408
  right=True,
1516
1409
  )
1517
1410
 
1518
- # Update used_conf configuration with epipolar a priori
1519
- # Add global min and max computed with grids
1520
- advanced_parameters.update_conf(
1521
- self.used_conf,
1522
- grid_correction_coef=self.pairs[pair_key][
1523
- "grid_correction_coef"
1524
- ],
1525
- pair_key=pair_key,
1526
- )
1411
+ # Update refined_conf configuration with epipolar a priori
1527
1412
  advanced_parameters.update_conf(
1528
- self.config_full_res,
1413
+ self.refined_conf,
1529
1414
  grid_correction_coef=self.pairs[pair_key][
1530
1415
  "grid_correction_coef"
1531
1416
  ],
1532
1417
  pair_key=pair_key,
1418
+ reference_dem=self.used_conf[INPUTS][
1419
+ sens_cst.INITIAL_ELEVATION
1420
+ ][sens_cst.DEM_PATH],
1533
1421
  )
1534
1422
  # saved used configuration
1535
- cars_dataset.save_dict(
1536
- self.used_conf,
1537
- os.path.join(self.out_dir, "used_conf.json"),
1538
- safe_save=True,
1539
- )
1423
+ self.save_configurations()
1540
1424
 
1541
1425
  # Generate min and max disp grids
1542
1426
  # Global disparity min and max will be computed from
@@ -1545,11 +1429,9 @@ class UnitPipeline(PipelineTemplate):
1545
1429
  self.dump_dir, "dense_matching", pair_key
1546
1430
  )
1547
1431
 
1548
- if (
1549
- self.which_resolution in ("first", "single")
1550
- and self.used_conf[ADVANCED][adv_cst.USE_EPIPOLAR_A_PRIORI]
1551
- is False
1552
- ):
1432
+ if self.which_resolution in ("first", "single") and self.used_conf[
1433
+ ADVANCED
1434
+ ][adv_cst.TERRAIN_A_PRIORI] in (None, {}):
1553
1435
  dmin = disp_min / self.res_resamp
1554
1436
  dmax = disp_max / self.res_resamp
1555
1437
  # generate_disparity_grids runs orchestrator.breakpoint()
@@ -1565,7 +1447,7 @@ class UnitPipeline(PipelineTemplate):
1565
1447
  )
1566
1448
  )
1567
1449
 
1568
- dsp_marg = self.sparse_mtch_sift_app.get_disparity_margin()
1450
+ dsp_marg = self.sparse_mtch_app.get_disparity_margin()
1569
1451
  updating_infos = {
1570
1452
  application_constants.APPLICATION_TAG: {
1571
1453
  sm_cst.DISPARITY_RANGE_COMPUTATION_TAG: {
@@ -1580,42 +1462,26 @@ class UnitPipeline(PipelineTemplate):
1580
1462
  self.cars_orchestrator.update_out_info(updating_infos)
1581
1463
 
1582
1464
  advanced_parameters.update_conf(
1583
- self.config_full_res,
1465
+ self.refined_conf,
1584
1466
  dmin=dmin,
1585
1467
  dmax=dmax,
1586
1468
  pair_key=pair_key,
1587
1469
  )
1588
1470
  else:
1589
- if None in (altitude_delta_min, altitude_delta_max):
1590
- # Generate min and max disp grids from dems
1591
- # generate_disparity_grids runs orchestrator.breakpoint()
1592
- self.pairs[pair_key]["disp_range_grid"] = (
1593
- self.dense_matching_app.generate_disparity_grids(
1594
- self.pairs[pair_key]["sensor_image_right"],
1595
- self.pairs[pair_key]["corrected_grid_right"],
1596
- self.geom_plugin_with_dem_and_geoid,
1597
- dem_min=dem_min,
1598
- dem_max=dem_max,
1599
- dem_median=dem_median,
1600
- pair_folder=dense_matching_pair_folder,
1601
- orchestrator=self.cars_orchestrator,
1602
- )
1603
- )
1604
- else:
1605
- # Generate min and max disp grids from deltas
1606
- # generate_disparity_grids runs orchestrator.breakpoint()
1607
- self.pairs[pair_key]["disp_range_grid"] = (
1608
- self.dense_matching_app.generate_disparity_grids(
1609
- self.pairs[pair_key]["sensor_image_right"],
1610
- self.pairs[pair_key]["corrected_grid_right"],
1611
- self.geom_plugin_with_dem_and_geoid,
1612
- altitude_delta_min=altitude_delta_min,
1613
- altitude_delta_max=altitude_delta_max,
1614
- dem_median=dem_median,
1615
- pair_folder=dense_matching_pair_folder,
1616
- orchestrator=self.cars_orchestrator,
1617
- )
1471
+ # Generate min and max disp grids from dems
1472
+ # generate_disparity_grids runs orchestrator.breakpoint()
1473
+ self.pairs[pair_key]["disp_range_grid"] = (
1474
+ self.dense_matching_app.generate_disparity_grids(
1475
+ self.pairs[pair_key]["sensor_image_right"],
1476
+ self.pairs[pair_key]["corrected_grid_right"],
1477
+ self.geom_plugin_with_dem_and_geoid,
1478
+ dem_min=dem_min,
1479
+ dem_max=dem_max,
1480
+ dem_median=dem_median,
1481
+ pair_folder=dense_matching_pair_folder,
1482
+ orchestrator=self.cars_orchestrator,
1618
1483
  )
1484
+ )
1619
1485
 
1620
1486
  if use_global_disp_range:
1621
1487
  # Generate min and max disp grids from constants
@@ -1631,7 +1497,7 @@ class UnitPipeline(PipelineTemplate):
1631
1497
  ]
1632
1498
 
1633
1499
  # update orchestrator_out_json
1634
- marg = self.sparse_mtch_sift_app.get_disparity_margin()
1500
+ marg = self.sparse_mtch_app.get_disparity_margin()
1635
1501
  updating_infos = {
1636
1502
  application_constants.APPLICATION_TAG: {
1637
1503
  sm_cst.DISPARITY_RANGE_COMPUTATION_TAG: {
@@ -1646,7 +1512,7 @@ class UnitPipeline(PipelineTemplate):
1646
1512
  self.cars_orchestrator.update_out_info(updating_infos)
1647
1513
 
1648
1514
  advanced_parameters.update_conf(
1649
- self.config_full_res,
1515
+ self.refined_conf,
1650
1516
  dmin=dmin,
1651
1517
  dmax=dmax,
1652
1518
  pair_key=pair_key,
@@ -1669,24 +1535,20 @@ class UnitPipeline(PipelineTemplate):
1669
1535
  # Update used_conf configuration with epipolar a priori
1670
1536
  # Add global min and max computed with grids
1671
1537
  advanced_parameters.update_conf(
1672
- self.used_conf,
1538
+ self.refined_conf,
1673
1539
  dmin=self.pairs[pair_key]["disp_range_grid"]["global_min"],
1674
1540
  dmax=self.pairs[pair_key]["disp_range_grid"]["global_max"],
1675
1541
  pair_key=pair_key,
1676
1542
  )
1677
1543
  advanced_parameters.update_conf(
1678
- self.config_full_res,
1544
+ self.refined_conf,
1679
1545
  dmin=self.pairs[pair_key]["disp_range_grid"]["global_min"],
1680
1546
  dmax=self.pairs[pair_key]["disp_range_grid"]["global_max"],
1681
1547
  pair_key=pair_key,
1682
1548
  )
1683
1549
 
1684
1550
  # saved used configuration
1685
- cars_dataset.save_dict(
1686
- self.used_conf,
1687
- os.path.join(self.out_dir, "used_conf.json"),
1688
- safe_save=True,
1689
- )
1551
+ self.save_configurations()
1690
1552
 
1691
1553
  # end of for loop, to finish computing disparity range grids
1692
1554
 
@@ -1743,6 +1605,10 @@ class UnitPipeline(PipelineTemplate):
1743
1605
  )
1744
1606
  )
1745
1607
 
1608
+ # Quick fix to reduce memory usage
1609
+ if self.res_resamp >= 16:
1610
+ optimum_tile_size = 200
1611
+
1746
1612
  # Run third epipolar resampling
1747
1613
  (
1748
1614
  new_epipolar_image_left,
@@ -1781,6 +1647,7 @@ class UnitPipeline(PipelineTemplate):
1781
1647
  geoid=self.used_conf[ADVANCED][adv_cst.GROUND_TRUTH_DSM][
1782
1648
  adv_cst.INPUT_GEOID
1783
1649
  ],
1650
+ scaling_coeff=self.scaling_coeff,
1784
1651
  )
1785
1652
  self.ground_truth_reprojection.run(
1786
1653
  self.pairs[pair_key]["sensor_image_left"],
@@ -1804,105 +1671,6 @@ class UnitPipeline(PipelineTemplate):
1804
1671
  ),
1805
1672
  )
1806
1673
 
1807
- # Run epipolar matching application
1808
- epipolar_disparity_map = self.dense_matching_app.run(
1809
- new_epipolar_image_left,
1810
- new_epipolar_image_right,
1811
- local_tile_optimal_size_fun,
1812
- orchestrator=self.cars_orchestrator,
1813
- pair_folder=os.path.join(
1814
- self.dump_dir, "dense_matching", pair_key
1815
- ),
1816
- pair_key=pair_key,
1817
- disp_range_grid=self.pairs[pair_key]["disp_range_grid"],
1818
- compute_disparity_masks=False,
1819
- margins_to_keep=(
1820
- self.pc_outlier_removal_1_app.get_epipolar_margin()
1821
- + self.pc_outlier_removal_2_app.get_epipolar_margin()
1822
- ),
1823
- texture_bands=texture_bands_indices,
1824
- )
1825
-
1826
- if self.quit_on_app("dense_matching"):
1827
- continue # keep iterating over pairs, but don't go further
1828
-
1829
- # Dense matches filling
1830
- if self.dense_match_filling_1.used_method == "plane":
1831
- # Fill holes in disparity map
1832
- (filled_with_1_epipolar_disparity_map) = (
1833
- self.dense_match_filling_1.run(
1834
- epipolar_disparity_map,
1835
- self.pairs[pair_key]["holes_bbox_left"],
1836
- self.pairs[pair_key]["holes_bbox_right"],
1837
- disp_min=self.pairs[pair_key]["disp_range_grid"][
1838
- "global_min"
1839
- ],
1840
- disp_max=np.max(
1841
- self.pairs[pair_key]["disp_range_grid"][
1842
- "global_max"
1843
- ]
1844
- ),
1845
- orchestrator=self.cars_orchestrator,
1846
- pair_folder=os.path.join(
1847
- self.dump_dir, "dense_match_filling_1", pair_key
1848
- ),
1849
- pair_key=pair_key,
1850
- )
1851
- )
1852
- else:
1853
- # Fill with zeros
1854
- (filled_with_1_epipolar_disparity_map) = (
1855
- self.dense_match_filling_1.run(
1856
- epipolar_disparity_map,
1857
- orchestrator=self.cars_orchestrator,
1858
- pair_folder=os.path.join(
1859
- self.dump_dir, "dense_match_filling_1", pair_key
1860
- ),
1861
- pair_key=pair_key,
1862
- )
1863
- )
1864
-
1865
- if self.quit_on_app("dense_match_filling.1"):
1866
- continue # keep iterating over pairs, but don't go further
1867
-
1868
- if self.dense_match_filling_2.used_method == "plane":
1869
- # Fill holes in disparity map
1870
- (filled_with_2_epipolar_disparity_map) = (
1871
- self.dense_match_filling_2.run(
1872
- filled_with_1_epipolar_disparity_map,
1873
- self.pairs[pair_key]["holes_bbox_left"],
1874
- self.pairs[pair_key]["holes_bbox_right"],
1875
- disp_min=self.pairs[pair_key]["disp_range_grid"][
1876
- "global_min"
1877
- ],
1878
- disp_max=np.max(
1879
- self.pairs[pair_key]["disp_range_grid"][
1880
- "global_max"
1881
- ]
1882
- ),
1883
- orchestrator=self.cars_orchestrator,
1884
- pair_folder=os.path.join(
1885
- self.dump_dir, "dense_match_filling_2", pair_key
1886
- ),
1887
- pair_key=pair_key,
1888
- )
1889
- )
1890
- else:
1891
- # Fill with zeros
1892
- (filled_with_2_epipolar_disparity_map) = (
1893
- self.dense_match_filling_2.run(
1894
- filled_with_1_epipolar_disparity_map,
1895
- orchestrator=self.cars_orchestrator,
1896
- pair_folder=os.path.join(
1897
- self.dump_dir, "dense_match_filling_2", pair_key
1898
- ),
1899
- pair_key=pair_key,
1900
- )
1901
- )
1902
-
1903
- if self.quit_on_app("dense_match_filling.2"):
1904
- continue # keep iterating over pairs, but don't go further
1905
-
1906
1674
  if self.epsg is None:
1907
1675
  # compute epsg
1908
1676
  # Epsg uses global disparity min and max
@@ -1924,6 +1692,131 @@ class UnitPipeline(PipelineTemplate):
1924
1692
  self.input_roi_poly, self.input_roi_epsg, self.epsg
1925
1693
  )
1926
1694
 
1695
+ self.vertical_crs = projection.get_output_crs(self.epsg, output)
1696
+
1697
+ if (
1698
+ self.save_output_dsm
1699
+ or self.save_output_point_cloud
1700
+ or self.dense_matching_app.get_method() == "auto"
1701
+ ):
1702
+ # Compute terrain bounding box /roi related to
1703
+ # current images
1704
+ (current_terrain_roi_bbox, intersection_poly) = (
1705
+ preprocessing.compute_terrain_bbox(
1706
+ self.pairs[pair_key]["sensor_image_left"],
1707
+ self.pairs[pair_key]["sensor_image_right"],
1708
+ new_epipolar_image_left,
1709
+ self.pairs[pair_key]["corrected_grid_left"],
1710
+ self.pairs[pair_key]["corrected_grid_right"],
1711
+ self.epsg,
1712
+ self.geom_plugin_with_dem_and_geoid,
1713
+ resolution=self.resolution,
1714
+ disp_min=self.pairs[pair_key]["disp_range_grid"][
1715
+ "global_min"
1716
+ ],
1717
+ disp_max=self.pairs[pair_key]["disp_range_grid"][
1718
+ "global_max"
1719
+ ],
1720
+ roi_poly=(
1721
+ None if self.debug_with_roi else self.roi_poly
1722
+ ),
1723
+ orchestrator=self.cars_orchestrator,
1724
+ pair_key=pair_key,
1725
+ pair_folder=os.path.join(
1726
+ self.dump_dir, "terrain_bbox", pair_key
1727
+ ),
1728
+ check_inputs=False,
1729
+ )
1730
+ )
1731
+ self.list_terrain_roi.append(current_terrain_roi_bbox)
1732
+ self.list_intersection_poly.append(intersection_poly)
1733
+
1734
+ # compute terrain bounds for later use
1735
+ (
1736
+ self.terrain_bounds,
1737
+ self.optimal_terrain_tile_width,
1738
+ ) = preprocessing.compute_terrain_bounds(
1739
+ self.list_terrain_roi,
1740
+ roi_poly=(None if self.debug_with_roi else self.roi_poly),
1741
+ resolution=self.resolution,
1742
+ )
1743
+
1744
+ if self.which_resolution not in ("final", "single"):
1745
+ self.terrain_bounds = dem_wrappers.modify_terrain_bounds(
1746
+ self.terrain_bounds,
1747
+ self.dem_generation_application.margin[0],
1748
+ self.dem_generation_application.margin[1],
1749
+ )
1750
+
1751
+ if self.dense_matching_app.get_method() == "auto":
1752
+ # Copy the initial corr_config in order to keep
1753
+ # the inputs that have already been checked
1754
+ corr_cfg = self.dense_matching_app.corr_config.copy()
1755
+
1756
+ # Find the conf that correspond to the land cover map
1757
+ conf = self.dense_matching_app.loader.find_auto_conf(
1758
+ intersection_poly,
1759
+ self.land_cover_map,
1760
+ self.classification_to_config_mapping,
1761
+ self.epsg,
1762
+ )
1763
+
1764
+ # Update the used_conf if order to reinitialize
1765
+ # the dense matching app
1766
+ # Because we kept the information regarding the ambiguity,
1767
+ # performance_map calculus..
1768
+ self.used_conf["applications"]["dense_matching"][
1769
+ "loader_conf"
1770
+ ] = conf
1771
+ self.used_conf["applications"]["dense_matching"][
1772
+ "method"
1773
+ ] = "custom"
1774
+
1775
+ # Re initialization of the dense matching application
1776
+ self.dense_matching_app = Application(
1777
+ "dense_matching",
1778
+ cfg=self.used_conf["applications"]["dense_matching"],
1779
+ )
1780
+
1781
+ # Update the corr_config with the inputs that have
1782
+ # already been checked
1783
+ self.dense_matching_app.corr_config["input"] = corr_cfg["input"]
1784
+
1785
+ # Run epipolar matching application
1786
+ epipolar_disparity_map = self.dense_matching_app.run(
1787
+ new_epipolar_image_left,
1788
+ new_epipolar_image_right,
1789
+ local_tile_optimal_size_fun,
1790
+ orchestrator=self.cars_orchestrator,
1791
+ pair_folder=os.path.join(
1792
+ self.dump_dir, "dense_matching", pair_key
1793
+ ),
1794
+ pair_key=pair_key,
1795
+ disp_range_grid=self.pairs[pair_key]["disp_range_grid"],
1796
+ compute_disparity_masks=False,
1797
+ margins_to_keep=sum(
1798
+ app.get_epipolar_margin()
1799
+ for _, app in self.pc_outlier_removal_apps.items()
1800
+ ),
1801
+ texture_bands=texture_bands_indices,
1802
+ )
1803
+
1804
+ if self.quit_on_app("dense_matching"):
1805
+ continue # keep iterating over pairs, but don't go further
1806
+
1807
+ # Fill with zeros
1808
+ (filled_epipolar_disparity_map) = self.dense_match_filling.run(
1809
+ epipolar_disparity_map,
1810
+ orchestrator=self.cars_orchestrator,
1811
+ pair_folder=os.path.join(
1812
+ self.dump_dir, "dense_match_filling", pair_key
1813
+ ),
1814
+ pair_key=pair_key,
1815
+ )
1816
+
1817
+ if self.quit_on_app("dense_match_filling"):
1818
+ continue # keep iterating over pairs, but don't go further
1819
+
1927
1820
  if isinstance(output[sens_cst.GEOID], str):
1928
1821
  output_geoid_path = output[sens_cst.GEOID]
1929
1822
  elif (
@@ -1943,7 +1836,6 @@ class UnitPipeline(PipelineTemplate):
1943
1836
  output_geoid_path = None
1944
1837
 
1945
1838
  depth_map_dir = None
1946
- last_depth_map_application = None
1947
1839
  if self.save_output_depth_map:
1948
1840
  depth_map_dir = os.path.join(
1949
1841
  self.out_dir, "depth_map", pair_key
@@ -1957,33 +1849,9 @@ class UnitPipeline(PipelineTemplate):
1957
1849
  )
1958
1850
  safe_makedirs(point_cloud_dir)
1959
1851
 
1960
- if self.save_output_depth_map or self.save_output_point_cloud:
1961
- if (
1962
- self.pc_outlier_removal_2_app.used_config.get(
1963
- "activated", False
1964
- )
1965
- is True
1966
- and self.merging is False
1967
- ):
1968
- last_depth_map_application = "pc_outlier_removal_2"
1969
- elif (
1970
- self.pc_outlier_removal_1_app.used_config.get(
1971
- "activated", False
1972
- )
1973
- is True
1974
- and self.merging is False
1975
- ):
1976
- last_depth_map_application = "pc_outlier_removal_1"
1977
- else:
1978
- last_depth_map_application = "triangulation"
1979
-
1980
1852
  triangulation_point_cloud_dir = (
1981
1853
  point_cloud_dir
1982
- if (
1983
- point_cloud_dir
1984
- and last_depth_map_application == "triangulation"
1985
- and self.merging is False
1986
- )
1854
+ if (point_cloud_dir and len(self.pc_outlier_removal_apps) == 0)
1987
1855
  else None
1988
1856
  )
1989
1857
 
@@ -1993,11 +1861,11 @@ class UnitPipeline(PipelineTemplate):
1993
1861
  self.pairs[pair_key]["sensor_image_right"],
1994
1862
  self.pairs[pair_key]["corrected_grid_left"],
1995
1863
  self.pairs[pair_key]["corrected_grid_right"],
1996
- filled_with_2_epipolar_disparity_map,
1864
+ filled_epipolar_disparity_map,
1997
1865
  self.geom_plugin_without_dem_and_geoid,
1998
1866
  new_epipolar_image_left,
1999
1867
  epsg=self.epsg,
2000
- denoising_overload_fun=denoising_overload_fun,
1868
+ denoising_overload_fun=None,
2001
1869
  source_pc_names=self.pairs_names,
2002
1870
  orchestrator=self.cars_orchestrator,
2003
1871
  pair_dump_dir=os.path.join(
@@ -2012,8 +1880,10 @@ class UnitPipeline(PipelineTemplate):
2012
1880
  ),
2013
1881
  depth_map_dir=depth_map_dir,
2014
1882
  point_cloud_dir=triangulation_point_cloud_dir,
2015
- save_output_coordinates=last_depth_map_application
2016
- == "triangulation",
1883
+ save_output_coordinates=(len(self.pc_outlier_removal_apps) == 0)
1884
+ and (
1885
+ self.save_output_depth_map or self.save_output_point_cloud
1886
+ ),
2017
1887
  save_output_color=bool(depth_map_dir)
2018
1888
  and self.auxiliary[out_cst.AUX_TEXTURE],
2019
1889
  save_output_classification=bool(depth_map_dir)
@@ -2031,158 +1901,49 @@ class UnitPipeline(PipelineTemplate):
2031
1901
  if self.quit_on_app("triangulation"):
2032
1902
  continue # keep iterating over pairs, but don't go further
2033
1903
 
2034
- if self.merging:
2035
- self.list_epipolar_point_clouds.append(epipolar_point_cloud)
2036
- else:
2037
- filtering_depth_map_dir = (
2038
- depth_map_dir
2039
- if (
2040
- depth_map_dir
2041
- and last_depth_map_application == "pc_outlier_removal_1"
2042
- )
2043
- else None
2044
- )
2045
- filtering_point_cloud_dir = (
2046
- point_cloud_dir
2047
- if (
2048
- point_cloud_dir
2049
- and last_depth_map_application == "pc_outlier_removal_1"
2050
- and self.merging is False
2051
- )
2052
- else None
2053
- )
1904
+ filtered_epipolar_point_cloud = epipolar_point_cloud
1905
+ for app_key, app in self.pc_outlier_removal_apps.items():
2054
1906
 
2055
- filtered_epipolar_point_cloud_1 = (
2056
- self.pc_outlier_removal_1_app.run(
2057
- epipolar_point_cloud,
2058
- depth_map_dir=filtering_depth_map_dir,
2059
- point_cloud_dir=filtering_point_cloud_dir,
2060
- dump_dir=os.path.join(
2061
- self.dump_dir, "pc_outlier_removal_1", pair_key
2062
- ),
2063
- epsg=self.epsg,
2064
- orchestrator=self.cars_orchestrator,
2065
- )
1907
+ app_key_is_last = (
1908
+ app_key == list(self.pc_outlier_removal_apps)[-1]
2066
1909
  )
2067
- if self.quit_on_app("point_cloud_outlier_removal.1"):
2068
- continue # keep iterating over pairs, but don't go further
2069
1910
  filtering_depth_map_dir = (
2070
- depth_map_dir
2071
- if (
2072
- depth_map_dir
2073
- and last_depth_map_application == "pc_outlier_removal_2"
2074
- )
2075
- else None
1911
+ depth_map_dir if app_key_is_last else None
2076
1912
  )
2077
1913
  filtering_point_cloud_dir = (
2078
- point_cloud_dir
2079
- if (
2080
- point_cloud_dir
2081
- and last_depth_map_application == "pc_outlier_removal_2"
2082
- and self.merging is False
2083
- )
2084
- else None
2085
- )
2086
- filtered_epipolar_point_cloud_2 = (
2087
- self.pc_outlier_removal_2_app.run(
2088
- filtered_epipolar_point_cloud_1,
2089
- depth_map_dir=filtering_depth_map_dir,
2090
- point_cloud_dir=filtering_point_cloud_dir,
2091
- dump_dir=os.path.join(
2092
- self.dump_dir, "pc_outlier_removal_2", pair_key
1914
+ point_cloud_dir if app_key_is_last else None
1915
+ )
1916
+
1917
+ filtered_epipolar_point_cloud = app.run(
1918
+ filtered_epipolar_point_cloud,
1919
+ depth_map_dir=filtering_depth_map_dir,
1920
+ point_cloud_dir=filtering_point_cloud_dir,
1921
+ dump_dir=os.path.join(
1922
+ self.dump_dir,
1923
+ ( # pylint: disable=inconsistent-quotes
1924
+ f"pc_outlier_removal"
1925
+ f"{str(app_key[27:]).replace('.', '_')}"
2093
1926
  ),
2094
- epsg=self.epsg,
2095
- orchestrator=self.cars_orchestrator,
2096
- )
2097
- )
2098
- if self.quit_on_app("point_cloud_outlier_removal.2"):
2099
- continue # keep iterating over pairs, but don't go further
2100
-
2101
- # denoising available only if we'll go further in the pipeline
2102
- if self.save_output_dsm or self.save_output_point_cloud:
2103
- denoised_epipolar_point_clouds = (
2104
- self.pc_denoising_application.run(
2105
- filtered_epipolar_point_cloud_2,
2106
- orchestrator=self.cars_orchestrator,
2107
- pair_folder=os.path.join(
2108
- self.dump_dir, "denoising", pair_key
2109
- ),
2110
- pair_key=pair_key,
2111
- )
2112
- )
2113
-
2114
- self.list_epipolar_point_clouds.append(
2115
- denoised_epipolar_point_clouds
2116
- )
2117
-
2118
- if self.quit_on_app("pc_denoising"):
2119
- # keep iterating over pairs, but don't go further
2120
- continue
2121
-
2122
- if self.save_output_dsm or self.save_output_point_cloud:
2123
- # Compute terrain bounding box /roi related to
2124
- # current images
2125
- (current_terrain_roi_bbox, intersection_poly) = (
2126
- preprocessing.compute_terrain_bbox(
2127
- self.pairs[pair_key]["sensor_image_left"],
2128
- self.pairs[pair_key]["sensor_image_right"],
2129
- new_epipolar_image_left,
2130
- self.pairs[pair_key]["corrected_grid_left"],
2131
- self.pairs[pair_key]["corrected_grid_right"],
2132
- self.epsg,
2133
- self.geom_plugin_with_dem_and_geoid,
2134
- resolution=self.resolution,
2135
- disp_min=self.pairs[pair_key]["disp_range_grid"][
2136
- "global_min"
2137
- ],
2138
- disp_max=self.pairs[pair_key]["disp_range_grid"][
2139
- "global_max"
2140
- ],
2141
- roi_poly=(
2142
- None if self.debug_with_roi else self.roi_poly
2143
- ),
2144
- orchestrator=self.cars_orchestrator,
2145
- pair_key=pair_key,
2146
- pair_folder=os.path.join(
2147
- self.dump_dir, "terrain_bbox", pair_key
2148
- ),
2149
- check_inputs=False,
2150
- )
2151
- )
2152
- self.list_terrain_roi.append(current_terrain_roi_bbox)
2153
- self.list_intersection_poly.append(intersection_poly)
2154
-
2155
- # compute terrain bounds for later use
2156
- (
2157
- self.terrain_bounds,
2158
- self.optimal_terrain_tile_width,
2159
- ) = preprocessing.compute_terrain_bounds(
2160
- self.list_terrain_roi,
2161
- roi_poly=(None if self.debug_with_roi else self.roi_poly),
2162
- resolution=self.resolution,
1927
+ pair_key,
1928
+ ),
1929
+ epsg=self.epsg,
1930
+ orchestrator=self.cars_orchestrator,
2163
1931
  )
1932
+ if self.quit_on_app("point_cloud_outlier_removal"):
1933
+ continue # keep iterating over pairs, but don't go further
2164
1934
 
2165
- if self.which_resolution not in ("final", "single"):
2166
- if self.dem_generation_roi is not None:
2167
- # To get the correct size for the dem generation
2168
- self.terrain_bounds = (
2169
- dem_wrappers.modify_terrain_bounds(
2170
- self.dem_generation_roi,
2171
- self.epsg,
2172
- self.dem_generation_application.margin,
2173
- )
2174
- )
1935
+ self.list_epipolar_point_clouds.append(
1936
+ filtered_epipolar_point_cloud
1937
+ )
2175
1938
 
2176
1939
  # quit if any app in the loop over the pairs was the last one
2177
1940
  # pylint:disable=too-many-boolean-expressions
2178
1941
  if (
2179
1942
  self.quit_on_app("dense_matching")
2180
- or self.quit_on_app("dense_match_filling.1")
2181
- or self.quit_on_app("dense_match_filling.2")
1943
+ or self.quit_on_app("dense_match_filling")
2182
1944
  or self.quit_on_app("triangulation")
2183
1945
  or self.quit_on_app("point_cloud_outlier_removal.1")
2184
1946
  or self.quit_on_app("point_cloud_outlier_removal.2")
2185
- or self.quit_on_app("pc_denoising")
2186
1947
  ):
2187
1948
  return True
2188
1949
 
@@ -2312,6 +2073,7 @@ class UnitPipeline(PipelineTemplate):
2312
2073
  _ = self.rasterization_application.run(
2313
2074
  self.point_cloud_to_rasterize,
2314
2075
  self.epsg,
2076
+ self.vertical_crs,
2315
2077
  resolution=self.resolution,
2316
2078
  orchestrator=self.cars_orchestrator,
2317
2079
  dsm_file_name=dsm_file_name,
@@ -2419,20 +2181,60 @@ class UnitPipeline(PipelineTemplate):
2419
2181
  sens_cst.DEM_PATH
2420
2182
  ]
2421
2183
  ),
2184
+ default_alt=self.geom_plugin_with_dem_and_geoid.default_alt,
2422
2185
  cars_orchestrator=self.cars_orchestrator,
2423
2186
  )
2424
2187
 
2188
+ # Update refined conf configuration with dem paths
2425
2189
  dem_median = paths["dem_median"]
2426
2190
  dem_min = paths["dem_min"]
2427
2191
  dem_max = paths["dem_max"]
2428
2192
 
2429
2193
  advanced_parameters.update_conf(
2430
- self.used_conf,
2194
+ self.refined_conf,
2431
2195
  dem_median=dem_median,
2432
2196
  dem_min=dem_min,
2433
2197
  dem_max=dem_max,
2434
2198
  )
2435
2199
 
2200
+ if self.used_conf[ADVANCED][USE_ENDOGENOUS_DEM]:
2201
+ # Generate new geom plugin with dem
2202
+ output_dem_dir = os.path.join(
2203
+ self.dump_dir, "initial_elevation"
2204
+ )
2205
+ new_geom_plugin = (
2206
+ sensor_inputs.generate_geometry_plugin_with_dem(
2207
+ self.geometry_plugin,
2208
+ self.used_conf[INPUTS],
2209
+ dem=dem_median,
2210
+ output_dem_dir=output_dem_dir,
2211
+ )
2212
+ )
2213
+
2214
+ for (
2215
+ pair_key,
2216
+ _,
2217
+ _,
2218
+ ) in self.list_sensor_pairs:
2219
+ new_grid_correction_coef = (
2220
+ self.generate_grid_correction_on_dem(
2221
+ pair_key,
2222
+ new_geom_plugin,
2223
+ )
2224
+ )
2225
+ if new_grid_correction_coef is not None:
2226
+ # Update refined_conf configuration with epipolar
2227
+ # a priori
2228
+ advanced_parameters.update_conf(
2229
+ self.refined_conf,
2230
+ grid_correction_coef=new_grid_correction_coef,
2231
+ pair_key=pair_key,
2232
+ reference_dem=dem_median,
2233
+ )
2234
+
2235
+ # saved used configuration
2236
+ self.save_configurations()
2237
+
2436
2238
  return False
2437
2239
 
2438
2240
  def filling(self): # noqa: C901 : too complex
@@ -2440,10 +2242,6 @@ class UnitPipeline(PipelineTemplate):
2440
2242
  Fill the dsm
2441
2243
  """
2442
2244
 
2443
- dsm_filling_1_dump_dir = os.path.join(self.dump_dir, "dsm_filling_1")
2444
- dsm_filling_2_dump_dir = os.path.join(self.dump_dir, "dsm_filling_2")
2445
- dsm_filling_3_dump_dir = os.path.join(self.dump_dir, "dsm_filling_3")
2446
-
2447
2245
  dsm_file_name = (
2448
2246
  os.path.join(
2449
2247
  self.out_dir,
@@ -2549,6 +2347,7 @@ class UnitPipeline(PipelineTemplate):
2549
2347
  )
2550
2348
 
2551
2349
  self.epsg = rasterio_get_epsg(dict_path["dsm"][0])
2350
+ self.vertical_crs = rasterio_get_crs(dict_path["dsm"][0])
2552
2351
 
2553
2352
  # Compute roi polygon, in input EPSG
2554
2353
  self.roi_poly = preprocessing.compute_roi_poly(
@@ -2678,47 +2477,56 @@ class UnitPipeline(PipelineTemplate):
2678
2477
  )
2679
2478
 
2680
2479
  # Project polygon if epsg is different
2681
- if self.epsg != inter_epsg:
2682
- inter_poly = projection.polygon_projection(
2683
- inter_poly, inter_epsg, self.epsg
2480
+ if self.vertical_crs != CRS(inter_epsg):
2481
+ inter_poly = projection.polygon_projection_crs(
2482
+ inter_poly, CRS(inter_epsg), self.vertical_crs
2684
2483
  )
2685
2484
 
2686
2485
  self.list_intersection_poly.append(inter_poly)
2687
2486
  else:
2688
2487
  self.list_intersection_poly = None
2689
2488
 
2690
- _ = self.dsm_filling_1_application.run(
2691
- dsm_file=dsm_file_name,
2692
- classif_file=classif_file_name,
2693
- filling_file=filling_file_name,
2694
- dump_dir=dsm_filling_1_dump_dir,
2695
- roi_polys=self.list_intersection_poly,
2696
- roi_epsg=self.epsg,
2697
- output_geoid=self.used_conf[OUTPUT][sens_cst.GEOID],
2698
- geom_plugin=self.geom_plugin_with_dem_and_geoid,
2699
- )
2700
-
2701
- if not self.dsm_filling_1_application.save_intermediate_data:
2702
- self.cars_orchestrator.add_to_clean(dsm_filling_1_dump_dir)
2703
-
2704
- if self.quit_on_app("dsm_filling.1"):
2705
- return True
2706
-
2707
- dtm_file_name = self.dsm_filling_2_application.run(
2708
- dsm_file=dsm_file_name,
2709
- classif_file=classif_file_name,
2710
- filling_file=filling_file_name,
2711
- dump_dir=dsm_filling_2_dump_dir,
2712
- roi_polys=self.list_intersection_poly,
2713
- roi_epsg=self.epsg,
2714
- orchestrator=self.cars_orchestrator,
2715
- )
2716
-
2717
- if not self.dsm_filling_2_application.save_intermediate_data:
2718
- self.cars_orchestrator.add_to_clean(dsm_filling_2_dump_dir)
2489
+ dtm_file_name = None
2490
+ for app_key, app in self.dsm_filling_apps.items():
2491
+
2492
+ app_dump_dir = os.path.join(
2493
+ self.dump_dir, app_key.replace(".", "_")
2494
+ )
2495
+
2496
+ if app.get_conf()["method"] == "exogenous_filling":
2497
+ _ = app.run(
2498
+ dsm_file=dsm_file_name,
2499
+ classif_file=classif_file_name,
2500
+ filling_file=filling_file_name,
2501
+ dump_dir=app_dump_dir,
2502
+ roi_polys=self.list_intersection_poly,
2503
+ roi_epsg=self.epsg,
2504
+ output_geoid=self.used_conf[OUTPUT][sens_cst.GEOID],
2505
+ geom_plugin=self.geom_plugin_with_dem_and_geoid,
2506
+ )
2507
+ elif app.get_conf()["method"] == "bulldozer":
2508
+ dtm_file_name = app.run(
2509
+ dsm_file=dsm_file_name,
2510
+ classif_file=classif_file_name,
2511
+ filling_file=filling_file_name,
2512
+ dump_dir=app_dump_dir,
2513
+ roi_polys=self.list_intersection_poly,
2514
+ roi_epsg=self.epsg,
2515
+ orchestrator=self.cars_orchestrator,
2516
+ )
2517
+ elif app.get_conf()["method"] == "border_interpolation":
2518
+ _ = app.run(
2519
+ dsm_file=dsm_file_name,
2520
+ classif_file=classif_file_name,
2521
+ filling_file=filling_file_name,
2522
+ dtm_file=dtm_file_name,
2523
+ dump_dir=app_dump_dir,
2524
+ roi_polys=self.list_intersection_poly,
2525
+ roi_epsg=self.epsg,
2526
+ )
2719
2527
 
2720
- if self.quit_on_app("dsm_filling.2"):
2721
- return True
2528
+ if not app.save_intermediate_data:
2529
+ self.cars_orchestrator.add_to_clean(app_dump_dir)
2722
2530
 
2723
2531
  _ = self.auxiliary_filling_application.run(
2724
2532
  dsm_file=dsm_file_name,
@@ -2729,199 +2537,31 @@ class UnitPipeline(PipelineTemplate):
2729
2537
  pairing=self.used_conf[INPUTS].get("pairing"),
2730
2538
  geom_plugin=self.geom_plugin_with_dem_and_geoid,
2731
2539
  texture_bands=self.texture_bands,
2540
+ output_geoid=self.used_conf[OUTPUT][sens_cst.GEOID],
2732
2541
  orchestrator=self.cars_orchestrator,
2733
2542
  )
2734
-
2735
- if self.quit_on_app("auxiliary_filling"):
2736
- return True
2737
-
2738
2543
  self.cars_orchestrator.breakpoint()
2739
2544
 
2740
- _ = self.dsm_filling_3_application.run(
2741
- dsm_file=dsm_file_name,
2742
- classif_file=classif_file_name,
2743
- filling_file=filling_file_name,
2744
- dtm_file=dtm_file_name,
2745
- dump_dir=dsm_filling_3_dump_dir,
2746
- roi_polys=self.list_intersection_poly,
2747
- roi_epsg=self.epsg,
2748
- )
2749
-
2750
- if not self.dsm_filling_3_application.save_intermediate_data:
2751
- self.cars_orchestrator.add_to_clean(dsm_filling_3_dump_dir)
2752
-
2753
- return self.quit_on_app("dsm_filling.3")
2545
+ return self.quit_on_app("auxiliary_filling")
2754
2546
 
2547
+ @cars_profile(name="Preprocess depth maps", interval=0.5)
2755
2548
  def preprocess_depth_maps(self):
2756
2549
  """
2757
2550
  Adds multiple processing steps to the depth maps :
2758
- Merging, denoising.
2551
+ Merging.
2759
2552
  Creates the point cloud that will be rasterized in
2760
2553
  the last step of the pipeline.
2761
2554
  """
2762
2555
 
2763
- if not self.merging:
2764
- self.point_cloud_to_rasterize = (
2765
- self.list_epipolar_point_clouds,
2766
- self.terrain_bounds,
2767
- )
2768
- self.color_type = self.point_cloud_to_rasterize[0][
2769
- 0
2770
- ].attributes.get("color_type", None)
2771
- else:
2772
- # find which application produce the final version of the
2773
- # point cloud. The last generated point cloud will be saved
2774
- # as official point cloud product if save_output_point_cloud
2775
- # is True.
2776
-
2777
- last_pc_application = None
2778
- # denoising application will produce a point cloud, unless
2779
- # it uses the 'none' method.
2780
- if self.pc_denoising_application.used_method != "none":
2781
- last_pc_application = "denoising"
2782
- else:
2783
- last_pc_application = "fusion"
2784
-
2785
- raster_app_margin = 0
2786
- if self.rasterization_application is not None:
2787
- raster_app_margin = self.rasterization_application.get_margins(
2788
- self.resolution
2789
- )
2790
-
2791
- merged_point_clouds = self.pc_fusion_application.run(
2792
- self.list_epipolar_point_clouds,
2793
- self.terrain_bounds,
2794
- self.epsg,
2795
- source_pc_names=(
2796
- self.pairs_names if self.compute_depth_map else None
2797
- ),
2798
- orchestrator=self.cars_orchestrator,
2799
- margins=raster_app_margin,
2800
- optimal_terrain_tile_width=self.optimal_terrain_tile_width,
2801
- roi=(self.roi_poly if self.debug_with_roi else None),
2802
- save_laz_output=self.save_output_point_cloud
2803
- and last_pc_application == "fusion",
2804
- )
2805
-
2806
- if self.quit_on_app("point_cloud_fusion"):
2807
- return True
2808
-
2809
- # denoise point cloud
2810
- denoised_merged_point_clouds = self.pc_denoising_application.run(
2811
- merged_point_clouds,
2812
- orchestrator=self.cars_orchestrator,
2813
- save_laz_output=self.save_output_point_cloud
2814
- and last_pc_application == "denoising",
2815
- )
2816
-
2817
- if self.quit_on_app("pc_denoising"):
2818
- return True
2819
-
2820
- # Rasterize merged and filtered point cloud
2821
- self.point_cloud_to_rasterize = denoised_merged_point_clouds
2822
-
2823
- # try getting the color type from multiple sources
2824
- self.color_type = self.list_epipolar_point_clouds[0].attributes.get(
2825
- "color_type",
2826
- self.point_cloud_to_rasterize.attributes.get(
2827
- "color_type", None
2828
- ),
2829
- )
2830
-
2831
- return False
2832
-
2833
- def load_input_depth_maps(self):
2834
- """
2835
- Loads all the data and creates all the variables used
2836
- later when processing a depth map, as if it was just computed.
2837
- """
2838
- # get epsg
2839
- self.epsg = self.used_conf[OUTPUT][out_cst.EPSG]
2840
-
2841
- output_parameters.intialize_product_index(
2842
- self.cars_orchestrator,
2843
- self.used_conf[OUTPUT]["product_level"],
2844
- self.used_conf[INPUTS][depth_cst.DEPTH_MAPS].keys(),
2845
- )
2846
-
2847
- # compute epsg
2848
- epsg_cloud = pc_fusion_wrappers.compute_epsg_from_point_cloud(
2849
- self.used_conf[INPUTS][depth_cst.DEPTH_MAPS]
2850
- )
2851
- if self.epsg is None:
2852
- self.epsg = epsg_cloud
2853
-
2854
- self.resolution = (
2855
- self.used_conf[OUTPUT][out_cst.RESOLUTION] * self.res_resamp
2556
+ self.point_cloud_to_rasterize = (
2557
+ self.list_epipolar_point_clouds,
2558
+ self.terrain_bounds,
2856
2559
  )
2857
-
2858
- # Compute roi polygon, in input EPSG
2859
- self.roi_poly = preprocessing.compute_roi_poly(
2860
- self.input_roi_poly, self.input_roi_epsg, self.epsg
2560
+ self.color_type = self.point_cloud_to_rasterize[0][0].attributes.get(
2561
+ "color_type", None
2861
2562
  )
2862
2563
 
2863
- if not self.merging:
2864
- # compute bounds
2865
- self.terrain_bounds = pc_fusion_wrappers.get_bounds(
2866
- self.used_conf[INPUTS][depth_cst.DEPTH_MAPS],
2867
- self.epsg,
2868
- roi_poly=self.roi_poly,
2869
- )
2870
-
2871
- self.list_epipolar_point_clouds = (
2872
- pc_fusion_algo.generate_point_clouds(
2873
- self.used_conf[INPUTS][depth_cst.DEPTH_MAPS],
2874
- self.cars_orchestrator,
2875
- tile_size=1000,
2876
- )
2877
- )
2878
- else:
2879
- # Compute terrain bounds and transform point clouds
2880
- (
2881
- self.terrain_bounds,
2882
- self.list_epipolar_point_clouds,
2883
- ) = pc_fusion_algo.transform_input_pc(
2884
- self.used_conf[INPUTS][depth_cst.DEPTH_MAPS],
2885
- self.epsg,
2886
- roi_poly=self.roi_poly,
2887
- epipolar_tile_size=1000, # TODO change it
2888
- orchestrator=self.cars_orchestrator,
2889
- )
2890
-
2891
- # Compute number of superposing point cloud for density
2892
- max_number_superposing_point_clouds = (
2893
- pc_fusion_wrappers.compute_max_nb_point_clouds(
2894
- self.list_epipolar_point_clouds
2895
- )
2896
- )
2897
-
2898
- # Compute average distance between two points
2899
- average_distance_point_cloud = (
2900
- pc_fusion_wrappers.compute_average_distance(
2901
- self.list_epipolar_point_clouds
2902
- )
2903
- )
2904
- self.optimal_terrain_tile_width = (
2905
- self.rasterization_application.get_optimal_tile_size(
2906
- self.cars_orchestrator.cluster.checked_conf_cluster[
2907
- "max_ram_per_worker"
2908
- ],
2909
- superposing_point_clouds=(
2910
- max_number_superposing_point_clouds
2911
- ),
2912
- point_cloud_resolution=average_distance_point_cloud,
2913
- )
2914
- )
2915
- # epsg_cloud and optimal_terrain_tile_width have the same epsg
2916
- self.optimal_terrain_tile_width = (
2917
- preprocessing.convert_optimal_tile_size_with_epsg(
2918
- self.terrain_bounds,
2919
- self.optimal_terrain_tile_width,
2920
- self.epsg,
2921
- epsg_cloud,
2922
- )
2923
- )
2924
-
2564
+ @cars_profile(name="Final cleanup", interval=0.5)
2925
2565
  def final_cleanup(self):
2926
2566
  """
2927
2567
  Clean temporary files and directory at the end of cars processing
@@ -2939,28 +2579,30 @@ class UnitPipeline(PipelineTemplate):
2939
2579
  not any(
2940
2580
  app.get("save_intermediate_data", False) is True
2941
2581
  for app in self.used_conf[APPLICATIONS].values()
2582
+ if app is not None
2942
2583
  )
2943
2584
  and not self.dsms_in_inputs
2944
2585
  ):
2945
2586
  self.cars_orchestrator.add_to_clean(self.dump_dir)
2946
2587
 
2947
- @cars_profile(name="run_dense_pipeline", interval=0.5)
2948
- def run(
2588
+ @cars_profile(name="run_unit_pipeline", interval=0.5)
2589
+ def run( # pylint: disable=too-many-positional-arguments
2949
2590
  self,
2950
- orchestrator_conf=None,
2951
2591
  generate_dems=False,
2952
2592
  which_resolution="single",
2953
2593
  use_sift_a_priori=False,
2954
2594
  first_res_out_dir=None,
2955
- final_out_dir=None,
2595
+ log_dir=None,
2956
2596
  ): # noqa C901
2957
2597
  """
2958
2598
  Run pipeline
2959
2599
 
2960
2600
  """
2601
+ if log_dir is not None:
2602
+ self.log_dir = log_dir
2603
+ else:
2604
+ self.log_dir = os.path.join(self.out_dir, "logs")
2961
2605
 
2962
- self.out_dir = self.used_conf[OUTPUT][out_cst.OUT_DIRECTORY]
2963
- self.dump_dir = os.path.join(self.out_dir, "dump_dir")
2964
2606
  self.first_res_out_dir = first_res_out_dir
2965
2607
  self.texture_bands = self.used_conf[ADVANCED][adv_cst.TEXTURE_BANDS]
2966
2608
 
@@ -2972,65 +2614,29 @@ class UnitPipeline(PipelineTemplate):
2972
2614
 
2973
2615
  self.which_resolution = which_resolution
2974
2616
 
2975
- # Save used conf
2976
- cars_dataset.save_dict(
2977
- self.used_conf,
2978
- os.path.join(self.out_dir, "used_conf.json"),
2979
- safe_save=True,
2980
- )
2981
-
2982
- if self.which_resolution not in ("single", "final"):
2983
- path_used_conf_res = (
2984
- "used_conf_res" + str(self.res_resamp) + ".json"
2985
- )
2986
- cars_dataset.save_dict(
2987
- self.used_conf,
2988
- os.path.join(final_out_dir, path_used_conf_res),
2989
- safe_save=True,
2990
- )
2991
-
2992
- if orchestrator_conf is None:
2993
- # start cars orchestrator
2994
- with orchestrator.Orchestrator(
2995
- orchestrator_conf=self.used_conf[ORCHESTRATOR],
2996
- out_dir=self.out_dir,
2997
- out_json_path=os.path.join(
2998
- self.out_dir,
2999
- out_cst.INFO_FILENAME,
3000
- ),
3001
- ) as self.cars_orchestrator:
3002
- # initialize out_json
3003
- self.cars_orchestrator.update_out_info({"version": __version__})
3004
-
3005
- if not self.dsms_in_inputs:
3006
- if self.compute_depth_map:
3007
- self.sensor_to_depth_maps()
3008
- else:
3009
- self.load_input_depth_maps()
3010
-
3011
- if self.save_output_dsm or self.save_output_point_cloud:
3012
- end_pipeline = self.preprocess_depth_maps()
3013
-
3014
- if self.save_output_dsm and not end_pipeline:
3015
- self.rasterize_point_cloud()
3016
- self.filling()
3017
- else:
3018
- self.filling()
3019
-
3020
- self.final_cleanup()
3021
- else:
3022
- self.cars_orchestrator = orchestrator_conf
2617
+ # saved used configuration
2618
+ self.save_configurations()
2619
+ # start cars orchestrator
2620
+ with orchestrator.Orchestrator(
2621
+ orchestrator_conf=self.used_conf[ORCHESTRATOR],
2622
+ out_dir=self.out_dir,
2623
+ log_dir=self.log_dir,
2624
+ out_json_path=os.path.join(
2625
+ self.out_dir,
2626
+ out_cst.INFO_FILENAME,
2627
+ ),
2628
+ ) as self.cars_orchestrator:
2629
+ # initialize out_json
2630
+ self.cars_orchestrator.update_out_info({"version": __version__})
3023
2631
 
3024
2632
  if not self.dsms_in_inputs:
3025
2633
  if self.compute_depth_map:
3026
2634
  self.sensor_to_depth_maps()
3027
- else:
3028
- self.load_input_depth_maps()
3029
2635
 
3030
2636
  if self.save_output_dsm or self.save_output_point_cloud:
3031
- end_pipeline = self.preprocess_depth_maps()
2637
+ self.preprocess_depth_maps()
3032
2638
 
3033
- if self.save_output_dsm and not end_pipeline:
2639
+ if self.save_output_dsm:
3034
2640
  self.rasterize_point_cloud()
3035
2641
  self.filling()
3036
2642
  else: