cars 1.0.0rc1__cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cars might be problematic. Click here for more details.

Files changed (200) hide show
  1. cars/__init__.py +74 -0
  2. cars/applications/__init__.py +37 -0
  3. cars/applications/application.py +117 -0
  4. cars/applications/application_constants.py +29 -0
  5. cars/applications/application_template.py +146 -0
  6. cars/applications/auxiliary_filling/__init__.py +29 -0
  7. cars/applications/auxiliary_filling/abstract_auxiliary_filling_app.py +104 -0
  8. cars/applications/auxiliary_filling/auxiliary_filling_algo.py +475 -0
  9. cars/applications/auxiliary_filling/auxiliary_filling_from_sensors_app.py +630 -0
  10. cars/applications/auxiliary_filling/auxiliary_filling_wrappers.py +90 -0
  11. cars/applications/dem_generation/__init__.py +30 -0
  12. cars/applications/dem_generation/abstract_dem_generation_app.py +116 -0
  13. cars/applications/dem_generation/bulldozer_config/base_config.yaml +42 -0
  14. cars/applications/dem_generation/bulldozer_dem_app.py +655 -0
  15. cars/applications/dem_generation/bulldozer_memory.py +55 -0
  16. cars/applications/dem_generation/dem_generation_algo.py +107 -0
  17. cars/applications/dem_generation/dem_generation_constants.py +32 -0
  18. cars/applications/dem_generation/dem_generation_wrappers.py +323 -0
  19. cars/applications/dense_match_filling/__init__.py +30 -0
  20. cars/applications/dense_match_filling/abstract_dense_match_filling_app.py +242 -0
  21. cars/applications/dense_match_filling/fill_disp_algo.py +113 -0
  22. cars/applications/dense_match_filling/fill_disp_constants.py +39 -0
  23. cars/applications/dense_match_filling/fill_disp_wrappers.py +83 -0
  24. cars/applications/dense_match_filling/zero_padding_app.py +302 -0
  25. cars/applications/dense_matching/__init__.py +30 -0
  26. cars/applications/dense_matching/abstract_dense_matching_app.py +261 -0
  27. cars/applications/dense_matching/census_mccnn_sgm_app.py +1460 -0
  28. cars/applications/dense_matching/cpp/__init__.py +0 -0
  29. cars/applications/dense_matching/cpp/dense_matching_cpp.cpython-312-i386-linux-gnu.so +0 -0
  30. cars/applications/dense_matching/cpp/dense_matching_cpp.py +94 -0
  31. cars/applications/dense_matching/cpp/includes/dense_matching.hpp +58 -0
  32. cars/applications/dense_matching/cpp/meson.build +9 -0
  33. cars/applications/dense_matching/cpp/src/bindings.cpp +13 -0
  34. cars/applications/dense_matching/cpp/src/dense_matching.cpp +207 -0
  35. cars/applications/dense_matching/dense_matching_algo.py +401 -0
  36. cars/applications/dense_matching/dense_matching_constants.py +89 -0
  37. cars/applications/dense_matching/dense_matching_wrappers.py +951 -0
  38. cars/applications/dense_matching/disparity_grid_algo.py +588 -0
  39. cars/applications/dense_matching/loaders/__init__.py +23 -0
  40. cars/applications/dense_matching/loaders/config_census_sgm_default.json +31 -0
  41. cars/applications/dense_matching/loaders/config_census_sgm_homogeneous.json +30 -0
  42. cars/applications/dense_matching/loaders/config_census_sgm_mountain_and_vegetation.json +30 -0
  43. cars/applications/dense_matching/loaders/config_census_sgm_shadow.json +30 -0
  44. cars/applications/dense_matching/loaders/config_census_sgm_sparse.json +36 -0
  45. cars/applications/dense_matching/loaders/config_census_sgm_urban.json +30 -0
  46. cars/applications/dense_matching/loaders/config_mapping.json +13 -0
  47. cars/applications/dense_matching/loaders/config_mccnn.json +28 -0
  48. cars/applications/dense_matching/loaders/global_land_cover_map.tif +0 -0
  49. cars/applications/dense_matching/loaders/pandora_loader.py +593 -0
  50. cars/applications/dsm_filling/__init__.py +32 -0
  51. cars/applications/dsm_filling/abstract_dsm_filling_app.py +101 -0
  52. cars/applications/dsm_filling/border_interpolation_app.py +270 -0
  53. cars/applications/dsm_filling/bulldozer_config/base_config.yaml +44 -0
  54. cars/applications/dsm_filling/bulldozer_filling_app.py +279 -0
  55. cars/applications/dsm_filling/exogenous_filling_app.py +333 -0
  56. cars/applications/grid_generation/__init__.py +30 -0
  57. cars/applications/grid_generation/abstract_grid_generation_app.py +142 -0
  58. cars/applications/grid_generation/epipolar_grid_generation_app.py +327 -0
  59. cars/applications/grid_generation/grid_correction_app.py +496 -0
  60. cars/applications/grid_generation/grid_generation_algo.py +388 -0
  61. cars/applications/grid_generation/grid_generation_constants.py +46 -0
  62. cars/applications/grid_generation/transform_grid.py +88 -0
  63. cars/applications/ground_truth_reprojection/__init__.py +30 -0
  64. cars/applications/ground_truth_reprojection/abstract_ground_truth_reprojection_app.py +137 -0
  65. cars/applications/ground_truth_reprojection/direct_localization_app.py +629 -0
  66. cars/applications/ground_truth_reprojection/ground_truth_reprojection_algo.py +275 -0
  67. cars/applications/point_cloud_outlier_removal/__init__.py +30 -0
  68. cars/applications/point_cloud_outlier_removal/abstract_outlier_removal_app.py +385 -0
  69. cars/applications/point_cloud_outlier_removal/outlier_removal_algo.py +392 -0
  70. cars/applications/point_cloud_outlier_removal/outlier_removal_constants.py +43 -0
  71. cars/applications/point_cloud_outlier_removal/small_components_app.py +527 -0
  72. cars/applications/point_cloud_outlier_removal/statistical_app.py +531 -0
  73. cars/applications/rasterization/__init__.py +30 -0
  74. cars/applications/rasterization/abstract_pc_rasterization_app.py +183 -0
  75. cars/applications/rasterization/rasterization_algo.py +534 -0
  76. cars/applications/rasterization/rasterization_constants.py +38 -0
  77. cars/applications/rasterization/rasterization_wrappers.py +634 -0
  78. cars/applications/rasterization/simple_gaussian_app.py +1152 -0
  79. cars/applications/resampling/__init__.py +28 -0
  80. cars/applications/resampling/abstract_resampling_app.py +187 -0
  81. cars/applications/resampling/bicubic_resampling_app.py +762 -0
  82. cars/applications/resampling/resampling_algo.py +614 -0
  83. cars/applications/resampling/resampling_constants.py +36 -0
  84. cars/applications/resampling/resampling_wrappers.py +309 -0
  85. cars/applications/sparse_matching/__init__.py +30 -0
  86. cars/applications/sparse_matching/abstract_sparse_matching_app.py +498 -0
  87. cars/applications/sparse_matching/sift_app.py +735 -0
  88. cars/applications/sparse_matching/sparse_matching_algo.py +360 -0
  89. cars/applications/sparse_matching/sparse_matching_constants.py +68 -0
  90. cars/applications/sparse_matching/sparse_matching_wrappers.py +238 -0
  91. cars/applications/triangulation/__init__.py +32 -0
  92. cars/applications/triangulation/abstract_triangulation_app.py +227 -0
  93. cars/applications/triangulation/line_of_sight_intersection_app.py +1243 -0
  94. cars/applications/triangulation/pc_transform.py +552 -0
  95. cars/applications/triangulation/triangulation_algo.py +371 -0
  96. cars/applications/triangulation/triangulation_constants.py +38 -0
  97. cars/applications/triangulation/triangulation_wrappers.py +259 -0
  98. cars/bundleadjustment.py +757 -0
  99. cars/cars.py +177 -0
  100. cars/conf/__init__.py +23 -0
  101. cars/conf/geoid/egm96.grd +0 -0
  102. cars/conf/geoid/egm96.grd.hdr +15 -0
  103. cars/conf/input_parameters.py +156 -0
  104. cars/conf/mask_cst.py +35 -0
  105. cars/core/__init__.py +23 -0
  106. cars/core/cars_logging.py +402 -0
  107. cars/core/constants.py +191 -0
  108. cars/core/constants_disparity.py +50 -0
  109. cars/core/datasets.py +140 -0
  110. cars/core/geometry/__init__.py +27 -0
  111. cars/core/geometry/abstract_geometry.py +1119 -0
  112. cars/core/geometry/shareloc_geometry.py +598 -0
  113. cars/core/inputs.py +568 -0
  114. cars/core/outputs.py +176 -0
  115. cars/core/preprocessing.py +722 -0
  116. cars/core/projection.py +843 -0
  117. cars/core/roi_tools.py +215 -0
  118. cars/core/tiling.py +774 -0
  119. cars/core/utils.py +164 -0
  120. cars/data_structures/__init__.py +23 -0
  121. cars/data_structures/cars_dataset.py +1541 -0
  122. cars/data_structures/cars_dict.py +74 -0
  123. cars/data_structures/corresponding_tiles_tools.py +186 -0
  124. cars/data_structures/dataframe_converter.py +185 -0
  125. cars/data_structures/format_transformation.py +297 -0
  126. cars/devibrate.py +689 -0
  127. cars/extractroi.py +264 -0
  128. cars/orchestrator/__init__.py +23 -0
  129. cars/orchestrator/achievement_tracker.py +125 -0
  130. cars/orchestrator/cluster/__init__.py +37 -0
  131. cars/orchestrator/cluster/abstract_cluster.py +244 -0
  132. cars/orchestrator/cluster/abstract_dask_cluster.py +375 -0
  133. cars/orchestrator/cluster/dask_cluster_tools.py +103 -0
  134. cars/orchestrator/cluster/dask_config/README.md +94 -0
  135. cars/orchestrator/cluster/dask_config/dask.yaml +21 -0
  136. cars/orchestrator/cluster/dask_config/distributed.yaml +70 -0
  137. cars/orchestrator/cluster/dask_config/jobqueue.yaml +26 -0
  138. cars/orchestrator/cluster/dask_config/reference_confs/dask-schema.yaml +137 -0
  139. cars/orchestrator/cluster/dask_config/reference_confs/dask.yaml +26 -0
  140. cars/orchestrator/cluster/dask_config/reference_confs/distributed-schema.yaml +1009 -0
  141. cars/orchestrator/cluster/dask_config/reference_confs/distributed.yaml +273 -0
  142. cars/orchestrator/cluster/dask_config/reference_confs/jobqueue.yaml +212 -0
  143. cars/orchestrator/cluster/dask_jobqueue_utils.py +204 -0
  144. cars/orchestrator/cluster/local_dask_cluster.py +116 -0
  145. cars/orchestrator/cluster/log_wrapper.py +1075 -0
  146. cars/orchestrator/cluster/mp_cluster/__init__.py +27 -0
  147. cars/orchestrator/cluster/mp_cluster/mp_factorizer.py +212 -0
  148. cars/orchestrator/cluster/mp_cluster/mp_objects.py +535 -0
  149. cars/orchestrator/cluster/mp_cluster/mp_tools.py +93 -0
  150. cars/orchestrator/cluster/mp_cluster/mp_wrapper.py +505 -0
  151. cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +873 -0
  152. cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py +399 -0
  153. cars/orchestrator/cluster/pbs_dask_cluster.py +207 -0
  154. cars/orchestrator/cluster/sequential_cluster.py +139 -0
  155. cars/orchestrator/cluster/slurm_dask_cluster.py +234 -0
  156. cars/orchestrator/orchestrator.py +905 -0
  157. cars/orchestrator/orchestrator_constants.py +29 -0
  158. cars/orchestrator/registry/__init__.py +23 -0
  159. cars/orchestrator/registry/abstract_registry.py +143 -0
  160. cars/orchestrator/registry/compute_registry.py +106 -0
  161. cars/orchestrator/registry/id_generator.py +116 -0
  162. cars/orchestrator/registry/replacer_registry.py +213 -0
  163. cars/orchestrator/registry/saver_registry.py +363 -0
  164. cars/orchestrator/registry/unseen_registry.py +118 -0
  165. cars/orchestrator/tiles_profiler.py +279 -0
  166. cars/pipelines/__init__.py +26 -0
  167. cars/pipelines/conf_resolution/conf_final_resolution.yaml +5 -0
  168. cars/pipelines/conf_resolution/conf_first_resolution.yaml +2 -0
  169. cars/pipelines/conf_resolution/conf_intermediate_resolution.yaml +2 -0
  170. cars/pipelines/default/__init__.py +26 -0
  171. cars/pipelines/default/default_pipeline.py +786 -0
  172. cars/pipelines/parameters/__init__.py +0 -0
  173. cars/pipelines/parameters/advanced_parameters.py +417 -0
  174. cars/pipelines/parameters/advanced_parameters_constants.py +69 -0
  175. cars/pipelines/parameters/application_parameters.py +71 -0
  176. cars/pipelines/parameters/depth_map_inputs.py +0 -0
  177. cars/pipelines/parameters/dsm_inputs.py +918 -0
  178. cars/pipelines/parameters/dsm_inputs_constants.py +25 -0
  179. cars/pipelines/parameters/output_constants.py +52 -0
  180. cars/pipelines/parameters/output_parameters.py +454 -0
  181. cars/pipelines/parameters/sensor_inputs.py +842 -0
  182. cars/pipelines/parameters/sensor_inputs_constants.py +49 -0
  183. cars/pipelines/parameters/sensor_loaders/__init__.py +29 -0
  184. cars/pipelines/parameters/sensor_loaders/basic_classif_loader.py +86 -0
  185. cars/pipelines/parameters/sensor_loaders/basic_image_loader.py +98 -0
  186. cars/pipelines/parameters/sensor_loaders/pivot_classif_loader.py +90 -0
  187. cars/pipelines/parameters/sensor_loaders/pivot_image_loader.py +105 -0
  188. cars/pipelines/parameters/sensor_loaders/sensor_loader.py +93 -0
  189. cars/pipelines/parameters/sensor_loaders/sensor_loader_template.py +71 -0
  190. cars/pipelines/parameters/sensor_loaders/slurp_classif_loader.py +86 -0
  191. cars/pipelines/pipeline.py +119 -0
  192. cars/pipelines/pipeline_constants.py +31 -0
  193. cars/pipelines/pipeline_template.py +139 -0
  194. cars/pipelines/unit/__init__.py +26 -0
  195. cars/pipelines/unit/unit_pipeline.py +2850 -0
  196. cars/starter.py +167 -0
  197. cars-1.0.0rc1.dist-info/METADATA +292 -0
  198. cars-1.0.0rc1.dist-info/RECORD +200 -0
  199. cars-1.0.0rc1.dist-info/WHEEL +6 -0
  200. cars-1.0.0rc1.dist-info/entry_points.txt +8 -0
@@ -0,0 +1,786 @@
1
+ #!/usr/bin/env python
2
+ # coding: utf8
3
+ #
4
+ # Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
5
+ #
6
+ # This file is part of CARS
7
+ # (see https://github.com/CNES/cars).
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ #
21
+ # pylint: disable=too-many-lines
22
+ # attribute-defined-outside-init is disabled so that we can create and use
23
+ # attributes however we need, to stick to the "everything is attribute" logic
24
+ # introduced in issue#895
25
+ # pylint: disable=attribute-defined-outside-init
26
+ # pylint: disable=too-many-nested-blocks
27
+ """
28
+ CARS default pipeline class file
29
+ """
30
+ # Standard imports
31
+ from __future__ import print_function
32
+
33
+ import copy
34
+ import logging
35
+ import os
36
+ import shutil
37
+ from collections import OrderedDict
38
+ from datetime import datetime
39
+
40
+ import yaml
41
+
42
+ # CARS imports
43
+ from cars.core import cars_logging
44
+ from cars.core.utils import safe_makedirs
45
+ from cars.data_structures import cars_dataset
46
+ from cars.orchestrator.cluster import log_wrapper
47
+ from cars.orchestrator.cluster.log_wrapper import cars_profile
48
+ from cars.pipelines.parameters import advanced_parameters
49
+ from cars.pipelines.parameters import advanced_parameters_constants as adv_cst
50
+ from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst
51
+ from cars.pipelines.parameters import output_constants as out_cst
52
+ from cars.pipelines.parameters import output_parameters
53
+ from cars.pipelines.parameters import sensor_inputs_constants as sens_cst
54
+ from cars.pipelines.pipeline import Pipeline
55
+ from cars.pipelines.pipeline_constants import (
56
+ ADVANCED,
57
+ APPLICATIONS,
58
+ INPUT,
59
+ ORCHESTRATOR,
60
+ OUTPUT,
61
+ )
62
+ from cars.pipelines.pipeline_template import PipelineTemplate
63
+ from cars.pipelines.unit.unit_pipeline import UnitPipeline
64
+
65
+ package_path = os.path.dirname(__file__)
66
+ FIRST_RES = "first_resolution"
67
+ INTERMEDIATE_RES = "intermediate_resolution"
68
+ FINAL_RES = "final_resolution"
69
+
70
+ PIPELINE_CONFS = {
71
+ FIRST_RES: os.path.join(
72
+ package_path,
73
+ "..",
74
+ "conf_resolution",
75
+ "conf_first_resolution.yaml",
76
+ ),
77
+ INTERMEDIATE_RES: os.path.join(
78
+ package_path,
79
+ "..",
80
+ "conf_resolution",
81
+ "conf_intermediate_resolution.yaml",
82
+ ),
83
+ FINAL_RES: os.path.join(
84
+ package_path,
85
+ "..",
86
+ "conf_resolution",
87
+ "conf_final_resolution.yaml",
88
+ ),
89
+ }
90
+
91
+
92
+ @Pipeline.register(
93
+ "default",
94
+ )
95
+ class DefaultPipeline(PipelineTemplate):
96
+ """
97
+ DefaultPipeline
98
+ """
99
+
100
+ # pylint: disable=too-many-instance-attributes
101
+
102
+ def __init__(self, conf, config_dir=None): # noqa: C901
103
+ """
104
+ Creates pipeline
105
+
106
+ :param pipeline_name: name of the pipeline.
107
+ :type pipeline_name: str
108
+ :param cfg: configuration {'matching_cost_method': value}
109
+ :type cfg: dictionary
110
+ :param config_dir: path to dir containing json or yaml file
111
+ :type config_dir: str
112
+ """
113
+
114
+ self.config_dir = config_dir
115
+ # Transform relative path to absolute path
116
+ if config_dir is not None:
117
+ config_dir = os.path.abspath(config_dir)
118
+
119
+ # Check global conf
120
+ self.check_global_schema(conf)
121
+
122
+ self.out_dir = conf[OUTPUT][out_cst.OUT_DIRECTORY]
123
+
124
+ # Get epipolar resolutions to use
125
+ self.epipolar_resolutions = (
126
+ advanced_parameters.get_epipolar_resolutions(conf.get(ADVANCED, {}))
127
+ )
128
+ if isinstance(self.epipolar_resolutions, int):
129
+ self.epipolar_resolutions = [self.epipolar_resolutions]
130
+
131
+ # Check application
132
+ self.check_applications(conf)
133
+ # Check input
134
+ conf[INPUT] = self.check_inputs(conf)
135
+ # check advanced
136
+ conf[ADVANCED] = self.check_advanced(conf)
137
+ # check output
138
+ conf[OUTPUT] = self.check_output(conf)
139
+
140
+ if dsm_cst.DSMS in conf[INPUT] and len(self.epipolar_resolutions) != 1:
141
+ logging.info(
142
+ "For the use of those pipelines, "
143
+ "you have to give only one resolution"
144
+ )
145
+ # overide epipolar resolutions
146
+ # TODO: delete with external dsm pipeline (refactoring)
147
+ self.epipolar_resolutions = [1]
148
+
149
+ used_configurations = {}
150
+ self.unit_pipelines = {}
151
+ self.positions = {}
152
+
153
+ self.intermediate_data_dir = os.path.join(
154
+ self.out_dir, "intermediate_data"
155
+ )
156
+
157
+ self.keep_low_res_dir = conf[ADVANCED][adv_cst.KEEP_LOW_RES_DIR]
158
+
159
+ # Get first res outdir for sift matches
160
+ self.first_res_out_dir_with_sensors = None
161
+
162
+ for epipolar_resolution_index, epipolar_res in enumerate(
163
+ self.epipolar_resolutions
164
+ ):
165
+ first_res = epipolar_resolution_index == 0
166
+ last_res = (
167
+ epipolar_resolution_index == len(self.epipolar_resolutions) - 1
168
+ )
169
+ intermediate_res = not first_res and not last_res
170
+
171
+ # set computed bool
172
+ self.positions[epipolar_resolution_index] = {
173
+ "first_res": first_res,
174
+ "intermediate_res": intermediate_res,
175
+ "last_res": last_res,
176
+ }
177
+
178
+ current_conf = copy.deepcopy(conf)
179
+ current_conf = extract_conf_with_resolution(
180
+ current_conf,
181
+ epipolar_res,
182
+ first_res,
183
+ intermediate_res,
184
+ last_res,
185
+ self.intermediate_data_dir,
186
+ )
187
+
188
+ if first_res:
189
+ self.first_res_out_dir_with_sensors = current_conf[OUTPUT][
190
+ "directory"
191
+ ]
192
+
193
+ if not isinstance(epipolar_res, int) or epipolar_res < 0:
194
+ raise RuntimeError("The resolution has to be an int > 0")
195
+
196
+ # Initialize unit pipeline in order to retrieve the
197
+ # used configuration
198
+ # This pipeline will not be run
199
+
200
+ current_unit_pipeline = UnitPipeline(
201
+ current_conf, config_dir=self.config_dir
202
+ )
203
+ self.unit_pipelines[epipolar_resolution_index] = (
204
+ current_unit_pipeline
205
+ )
206
+ # Get used_conf
207
+ used_configurations[epipolar_res] = current_unit_pipeline.used_conf
208
+
209
+ # Generate full used_conf
210
+ full_used_conf = merge_used_conf(
211
+ used_configurations, self.epipolar_resolutions
212
+ )
213
+ # Save used_conf
214
+ cars_dataset.save_dict(
215
+ full_used_conf,
216
+ os.path.join(self.out_dir, "global_used_conf.yaml"),
217
+ )
218
+
219
+ def check_inputs(self, conf, config_json_dir=None):
220
+ """
221
+ Check the inputs given
222
+
223
+ :param conf: configuration
224
+ :type conf: dict
225
+ :param config_dir: directory of used json, if
226
+ user filled paths with relative paths
227
+ :type config_dir: str
228
+
229
+ :return: overloader inputs
230
+ :rtype: dict
231
+ """
232
+ return UnitPipeline.check_inputs(
233
+ conf[INPUT], config_dir=self.config_dir
234
+ )
235
+
236
+ def check_output(self, conf):
237
+ """
238
+ Check the output given
239
+
240
+ :param conf: configuration of output
241
+ :type conf: dict
242
+
243
+ :return overloader output
244
+ :rtype : dict
245
+ """
246
+ conf_output, self.scaling_coeff = (
247
+ output_parameters.check_output_parameters(
248
+ conf[INPUT], conf[OUTPUT], self.scaling_coeff
249
+ )
250
+ )
251
+ return conf_output
252
+
253
+ def check_advanced(self, conf):
254
+ """
255
+ Check all conf for advanced configuration
256
+
257
+ :return: overridden advanced conf
258
+ :rtype: dict
259
+ """
260
+ (_, advanced, _, _, _, self.scaling_coeff, _, _) = (
261
+ advanced_parameters.check_advanced_parameters(
262
+ conf[INPUT],
263
+ conf.get(ADVANCED, {}),
264
+ check_epipolar_a_priori=True,
265
+ )
266
+ )
267
+ return advanced
268
+
269
+ def check_applications(self, conf):
270
+ """
271
+ Check the given configuration for applications
272
+
273
+ :param conf: configuration of applications
274
+ :type conf: dict
275
+ """
276
+ applications_conf = conf.get(APPLICATIONS, {})
277
+ # check format: contains "all" of "resolutions
278
+
279
+ int_keys = [int(epi_res) for epi_res in self.epipolar_resolutions]
280
+ string_keys = [str(key) for key in int_keys]
281
+
282
+ possible_keys = ["all"] + int_keys + string_keys
283
+
284
+ # Check conf keys in possible keys
285
+ for app_base_key in applications_conf.keys():
286
+ if app_base_key not in possible_keys:
287
+ raise RuntimeError(
288
+ "Application key {} not in possibles keys in : 'all', {} , "
289
+ "as int or str".format(app_base_key, string_keys)
290
+ )
291
+
292
+ # Key str and int key are not defined for the same resolution
293
+ for resolution in int_keys:
294
+ if (
295
+ resolution in applications_conf
296
+ and str(resolution) in applications_conf
297
+ ):
298
+ raise RuntimeError(
299
+ "Application configuration for {} resolution "
300
+ "is defined both "
301
+ "with int and str key".format(resolution)
302
+ )
303
+
304
+ def cleanup_low_res_dir(self):
305
+ """
306
+ Clean low res dir
307
+ """
308
+
309
+ if os.path.exists(self.intermediate_data_dir) and os.path.isdir(
310
+ self.intermediate_data_dir
311
+ ):
312
+ try:
313
+ shutil.rmtree(self.intermediate_data_dir)
314
+ logging.info(
315
+ f"th directory {self.intermediate_data_dir} "
316
+ f" has been cleaned."
317
+ )
318
+ except Exception as exception:
319
+ logging.error(
320
+ f"Error while deleting {self.intermediate_data_dir}: "
321
+ f"{exception}"
322
+ )
323
+ else:
324
+ logging.info(
325
+ f"The directory {self.intermediate_data_dir} has not "
326
+ f"been deleted"
327
+ )
328
+
329
+ def override_with_apriori(self, conf, previous_out_dir, first_res):
330
+ """
331
+ Override configuration with terrain a priori
332
+
333
+ :param new_conf: configuration
334
+ :type new_conf: dict
335
+ """
336
+
337
+ new_conf = copy.deepcopy(conf)
338
+
339
+ # Extract avanced parameters configuration
340
+ # epipolar and terrain a priori can only be used on first resolution
341
+ if not first_res:
342
+ dem_min = os.path.join(previous_out_dir, "dsm/dem_min.tif")
343
+ dem_max = os.path.join(previous_out_dir, "dsm/dem_max.tif")
344
+ dem_median = os.path.join(previous_out_dir, "dsm/dem_median.tif")
345
+
346
+ new_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] = {
347
+ "dem_min": dem_min,
348
+ "dem_max": dem_max,
349
+ "dem_median": dem_median,
350
+ }
351
+ # Use initial elevation or dem median according to use wish
352
+ if new_conf[INPUT][sens_cst.INITIAL_ELEVATION]["dem"] is None:
353
+ new_conf[INPUT][sens_cst.INITIAL_ELEVATION] = dem_median
354
+ else:
355
+ new_conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI]["dem_median"] = (
356
+ new_conf[INPUT][sens_cst.INITIAL_ELEVATION]["dem"]
357
+ )
358
+ if new_conf[ADVANCED][adv_cst.USE_ENDOGENOUS_DEM] and not first_res:
359
+ new_conf[INPUT][sens_cst.INITIAL_ELEVATION] = dem_median
360
+
361
+ new_conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI] = None
362
+
363
+ return new_conf
364
+
365
+ @cars_profile(name="Run_default_pipeline", interval=0.5)
366
+ def run(self, args=None): # noqa C901
367
+ """
368
+ Run pipeline
369
+
370
+ """
371
+
372
+ global_log_file = os.path.join(
373
+ self.out_dir,
374
+ "logs",
375
+ "{}_{}.log".format(
376
+ datetime.now().strftime("%y-%m-%d_%Hh%Mm"), "default_pipeline"
377
+ ),
378
+ )
379
+
380
+ previous_out_dir = None
381
+ updated_conf = {}
382
+ for resolution_index, epipolar_res in enumerate(
383
+ self.epipolar_resolutions
384
+ ):
385
+
386
+ # Get tested unit pipeline
387
+ used_pipeline = self.unit_pipelines[resolution_index]
388
+ current_out_dir = used_pipeline.used_conf[OUTPUT]["directory"]
389
+
390
+ # get position
391
+ first_res, _, last_res = (
392
+ self.positions[resolution_index]["first_res"],
393
+ self.positions[resolution_index]["intermediate_res"],
394
+ self.positions[resolution_index]["last_res"],
395
+ )
396
+
397
+ # setup logging
398
+ loglevel = getattr(args, "loglevel", "PROGRESS").upper()
399
+
400
+ current_log_dir = os.path.join(
401
+ self.out_dir, "logs", "res_" + str(epipolar_res)
402
+ )
403
+
404
+ cars_logging.setup_logging(
405
+ loglevel,
406
+ out_dir=current_log_dir,
407
+ pipeline="unit_pipeline",
408
+ global_log_file=global_log_file,
409
+ )
410
+
411
+ cars_logging.add_progress_message(
412
+ "Starting pipeline for resolution 1/" + str(epipolar_res)
413
+ )
414
+
415
+ # use sift a priori if not first
416
+ use_sift_a_priori = False
417
+ if not first_res:
418
+ use_sift_a_priori = True
419
+
420
+ # define wich resolution
421
+ if first_res and last_res:
422
+ which_resolution = "single"
423
+ elif first_res:
424
+ which_resolution = "first"
425
+ elif last_res:
426
+ which_resolution = "final"
427
+ else:
428
+ which_resolution = "intermediate"
429
+
430
+ # Generate dem
431
+ generate_dems = True
432
+ if last_res:
433
+ generate_dems = False
434
+
435
+ # Overide with a priori
436
+ overridden_conf = self.override_with_apriori(
437
+ used_pipeline.used_conf, previous_out_dir, first_res
438
+ )
439
+ updated_pipeline = UnitPipeline(
440
+ overridden_conf, config_dir=self.config_dir
441
+ )
442
+ updated_pipeline.run(
443
+ generate_dems=generate_dems,
444
+ which_resolution=which_resolution,
445
+ use_sift_a_priori=use_sift_a_priori,
446
+ first_res_out_dir=self.first_res_out_dir_with_sensors,
447
+ log_dir=current_log_dir,
448
+ )
449
+
450
+ # update previous out dir
451
+ previous_out_dir = current_out_dir
452
+
453
+ # generate summary
454
+ log_wrapper.generate_summary(
455
+ current_log_dir, updated_pipeline.used_conf
456
+ )
457
+
458
+ updated_conf[epipolar_res] = updated_pipeline.used_conf
459
+
460
+ # Generate full used_conf
461
+ full_used_conf = merge_used_conf(
462
+ updated_conf, self.epipolar_resolutions
463
+ )
464
+ # Save used_conf
465
+ cars_dataset.save_dict(
466
+ full_used_conf,
467
+ os.path.join(self.out_dir, "global_used_conf.yaml"),
468
+ )
469
+
470
+ # Merge profiling in pdf
471
+ log_wrapper.generate_pdf_profiling(os.path.join(self.out_dir, "logs"))
472
+
473
+ # clean outdir
474
+ if not self.keep_low_res_dir:
475
+ self.cleanup_low_res_dir()
476
+
477
+
478
+ def extract_applications(
479
+ current_applications_conf, res, default_conf_for_res, filling_applications
480
+ ):
481
+ """
482
+ Extract applications for current resolution
483
+
484
+ :param current_applications_conf: current applications configuration
485
+ :type current_applications_conf: dict
486
+ :param res: resolution to extract
487
+ :type res: int
488
+ :param default_conf_for_res: default configuration for resolution
489
+ :type default_conf_for_res: dict
490
+ :param filling_applications: filling applications configuration
491
+ :type filling_applications: dict
492
+
493
+ :return: configuration for the given resolution
494
+ :rtype: dict
495
+ """
496
+
497
+ # "all" : applied to all conf
498
+ # int (1, 2, 4, 8, 16, ...) applied for specified resolution
499
+
500
+ all_conf = current_applications_conf.get("all", {})
501
+ # Overide with default_conf_for_res
502
+ all_conf = overide_pipeline_conf(all_conf, default_conf_for_res)
503
+ # Get configuration for current res
504
+ if res in current_applications_conf:
505
+ # key is int
506
+ key = res
507
+ else:
508
+ key = str(res)
509
+
510
+ res_conf = current_applications_conf.get(key, {})
511
+
512
+ # Overide all conf with current res conf
513
+ new_application_conf = overide_pipeline_conf(all_conf, res_conf)
514
+
515
+ # Overide with filling applications
516
+ new_application_conf = overide_pipeline_conf(
517
+ new_application_conf, filling_applications, append_classification=True
518
+ )
519
+ return new_application_conf
520
+
521
+
522
+ # pylint: disable=too-many-positional-arguments
523
+ def extract_conf_with_resolution(
524
+ current_conf,
525
+ res,
526
+ first_res,
527
+ intermediate_res,
528
+ last_res,
529
+ intermediate_data_dir,
530
+ ):
531
+ """
532
+ Extract the configuration for the given resolution
533
+
534
+ :param current_conf: current configuration
535
+ :type current_conf: dict
536
+ :param res: resolution to extract
537
+ :type res: int
538
+ :return: configuration for the given resolution
539
+ :rtype: dict
540
+ :param first_res: is first resolution
541
+ :type first_res: bool
542
+ :param intermediate_res: is intermediate resolution
543
+ :type intermediate_res: bool
544
+ :param last_res: is last resolution
545
+ :type last_res: bool
546
+ :param previous_out_dir: path to previous outdir
547
+ :type: previous_out_dir: str
548
+ """
549
+
550
+ new_dir_out_dir = current_conf[OUTPUT][out_cst.OUT_DIRECTORY]
551
+ if not last_res:
552
+ new_dir_out_dir = os.path.join(
553
+ intermediate_data_dir, "out_res" + str(res)
554
+ )
555
+ safe_makedirs(new_dir_out_dir)
556
+
557
+ new_conf = copy.deepcopy(current_conf)
558
+
559
+ # Get save intermediate data
560
+ if isinstance(new_conf[ADVANCED][adv_cst.SAVE_INTERMEDIATE_DATA], dict):
561
+ # If save_intermediate_data is not a dict, we set it to False
562
+ new_conf[ADVANCED][adv_cst.SAVE_INTERMEDIATE_DATA] = new_conf[ADVANCED][
563
+ adv_cst.SAVE_INTERMEDIATE_DATA
564
+ ].get("resolution_" + str(res), False)
565
+
566
+ # Overide epipolar resolution
567
+ new_conf[ADVANCED][adv_cst.EPIPOLAR_RESOLUTIONS] = res
568
+
569
+ # Overide configuration with pipeline conf
570
+ if first_res:
571
+ # read the first resolution conf with json package
572
+ with open(PIPELINE_CONFS[FIRST_RES], "r", encoding="utf-8") as file:
573
+ overiding_conf = yaml.safe_load(file)
574
+ elif intermediate_res:
575
+ with open(
576
+ PIPELINE_CONFS[INTERMEDIATE_RES], "r", encoding="utf-8"
577
+ ) as file:
578
+ overiding_conf = yaml.safe_load(file)
579
+ else:
580
+ with open(PIPELINE_CONFS[FINAL_RES], "r", encoding="utf-8") as file:
581
+ overiding_conf = yaml.safe_load(file)
582
+
583
+ if last_res and dsm_cst.DSMS not in current_conf[INPUT]:
584
+ # Use filling applications only for last resolution
585
+ filling_applications = generate_filling_applications(
586
+ current_conf[INPUT]
587
+ )
588
+ else:
589
+ filling_applications = {}
590
+
591
+ # extract application
592
+ new_conf[APPLICATIONS] = extract_applications(
593
+ current_conf.get(APPLICATIONS, {}),
594
+ res,
595
+ overiding_conf.get(APPLICATIONS, {}),
596
+ filling_applications,
597
+ )
598
+
599
+ # Overide output to not compute data
600
+ # Overide resolution to let unit pipeline manage it
601
+ if not last_res:
602
+ overiding_conf = {
603
+ OUTPUT: {
604
+ out_cst.OUT_DIRECTORY: new_dir_out_dir,
605
+ out_cst.RESOLUTION: None,
606
+ out_cst.SAVE_BY_PAIR: True,
607
+ out_cst.AUXILIARY: {
608
+ out_cst.AUX_DEM_MAX: True,
609
+ out_cst.AUX_DEM_MIN: True,
610
+ out_cst.AUX_DEM_MEDIAN: True,
611
+ },
612
+ },
613
+ APPLICATIONS: {
614
+ "dense_matching": {
615
+ "performance_map_method": ["risk", "intervals"]
616
+ }
617
+ },
618
+ }
619
+ new_conf = overide_pipeline_conf(new_conf, overiding_conf)
620
+
621
+ # set product level to dsm
622
+ new_conf[OUTPUT][out_cst.PRODUCT_LEVEL] = ["dsm"]
623
+ # remove resolution to let CARS compute it for current
624
+ # epipolar resolution
625
+ new_conf[OUTPUT]["resolution"] = None
626
+
627
+ if not new_conf[ADVANCED][adv_cst.SAVE_INTERMEDIATE_DATA]:
628
+ # Save the less possible things
629
+ aux_items = new_conf[OUTPUT][out_cst.AUXILIARY].items()
630
+ for aux_key, _ in aux_items:
631
+ if aux_key not in ("dem_min", "dem_max", "dem_median", "image"):
632
+ new_conf[OUTPUT][out_cst.AUXILIARY][aux_key] = False
633
+
634
+ return new_conf
635
+
636
+
637
+ def generate_filling_applications(inputs_conf):
638
+ """
639
+ Generate filling applications configuration according to inputs
640
+
641
+ :param inputs_conf: inputs configuration
642
+ :type inputs_conf: dict
643
+ """
644
+
645
+ filling_applications = {}
646
+
647
+ # Generate applications configuration
648
+ for filling_name, classif_values in inputs_conf[sens_cst.FILLING].items():
649
+ # No filling
650
+ if classif_values is None:
651
+ continue
652
+
653
+ classif_values = list(map(str, classif_values))
654
+
655
+ # Update application configuration
656
+ if filling_name == "fill_with_geoid":
657
+ new_filling_conf = {
658
+ "dense_match_filling": {
659
+ "method": "zero_padding",
660
+ "classification": classif_values,
661
+ },
662
+ "dsm_filling.1": {
663
+ "method": "exogenous_filling",
664
+ "classification": classif_values,
665
+ "fill_with_geoid": classif_values,
666
+ },
667
+ }
668
+ elif filling_name == "interpolate_from_borders":
669
+ new_filling_conf = {
670
+ "dense_match_filling": {
671
+ "method": "zero_padding",
672
+ "classification": classif_values,
673
+ },
674
+ "dsm_filling.2": {
675
+ "method": "bulldozer",
676
+ "classification": classif_values,
677
+ },
678
+ "dsm_filling.3": {
679
+ "method": "border_interpolation",
680
+ "classification": classif_values,
681
+ },
682
+ }
683
+ elif filling_name == "fill_with_endogenous_dem":
684
+ new_filling_conf = {
685
+ "dense_match_filling": {
686
+ "method": "zero_padding",
687
+ "classification": classif_values,
688
+ },
689
+ "dsm_filling.1": {
690
+ "method": "exogenous_filling",
691
+ "classification": classif_values,
692
+ },
693
+ "dsm_filling.2": {
694
+ "method": "bulldozer",
695
+ "classification": classif_values,
696
+ },
697
+ }
698
+ elif filling_name == "fill_with_exogenous_dem":
699
+ new_filling_conf = {
700
+ "dense_match_filling": {
701
+ "method": "zero_padding",
702
+ "classification": classif_values,
703
+ },
704
+ "dsm_filling.2": {
705
+ "method": "bulldozer",
706
+ "classification": classif_values,
707
+ },
708
+ }
709
+ else:
710
+ new_filling_conf = {}
711
+
712
+ # Update application configuration
713
+ filling_applications = overide_pipeline_conf(
714
+ filling_applications, new_filling_conf, append_classification=True
715
+ )
716
+
717
+ return filling_applications
718
+
719
+
720
+ def overide_pipeline_conf(conf, overiding_conf, append_classification=False):
721
+ """
722
+ Merge two dictionaries recursively without removing keys from the base conf.
723
+
724
+ :param conf: base configuration dictionary
725
+ :type conf: dict
726
+ :param overiding_conf: overriding configuration dictionary
727
+ :type overiding_conf: dict
728
+ :return: merged configuration
729
+ :rtype: dict
730
+ """
731
+ result = copy.deepcopy(conf)
732
+
733
+ def merge_recursive(base_dict, override_dict):
734
+ """
735
+ Main recursive function
736
+ """
737
+ for key, value in override_dict.items():
738
+ if (
739
+ key in base_dict
740
+ and isinstance(base_dict[key], dict)
741
+ and isinstance(value, dict)
742
+ ):
743
+ merge_recursive(base_dict[key], value)
744
+ elif (
745
+ append_classification
746
+ and key in base_dict
747
+ and isinstance(base_dict[key], list)
748
+ and isinstance(value, list)
749
+ and key == "classification"
750
+ ):
751
+ # extend list, avoiding duplicates
752
+ base_dict[key] = list(
753
+ OrderedDict.fromkeys(base_dict[key] + value)
754
+ )
755
+ else:
756
+ base_dict[key] = value
757
+
758
+ merge_recursive(result, overiding_conf)
759
+ return result
760
+
761
+
762
+ def merge_used_conf(used_configurations, epipolar_resolutions):
763
+ """
764
+ Merge all used configuration
765
+ """
766
+ used_configurations = copy.deepcopy(used_configurations)
767
+
768
+ merged_conf = {
769
+ INPUT: used_configurations[epipolar_resolutions[0]][INPUT],
770
+ ADVANCED: used_configurations[epipolar_resolutions[0]][ADVANCED],
771
+ OUTPUT: used_configurations[epipolar_resolutions[0]][OUTPUT],
772
+ ORCHESTRATOR: used_configurations[epipolar_resolutions[0]][
773
+ ORCHESTRATOR
774
+ ],
775
+ }
776
+
777
+ merged_conf[APPLICATIONS] = {}
778
+ merged_conf[APPLICATIONS]["all"] = {}
779
+
780
+ # Merge applications
781
+ for res in epipolar_resolutions:
782
+ merged_conf[APPLICATIONS][res] = used_configurations[res][APPLICATIONS]
783
+
784
+ # apply epipolar resolutions
785
+ merged_conf[ADVANCED]["epipolar_resolutions"] = epipolar_resolutions
786
+ return merged_conf