cars 1.0.0rc2__cp312-cp312-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cars might be problematic. Click here for more details.

Files changed (225) hide show
  1. cars/__init__.py +86 -0
  2. cars/applications/__init__.py +40 -0
  3. cars/applications/application.py +117 -0
  4. cars/applications/application_constants.py +29 -0
  5. cars/applications/application_template.py +146 -0
  6. cars/applications/auxiliary_filling/__init__.py +29 -0
  7. cars/applications/auxiliary_filling/abstract_auxiliary_filling_app.py +105 -0
  8. cars/applications/auxiliary_filling/auxiliary_filling_algo.py +475 -0
  9. cars/applications/auxiliary_filling/auxiliary_filling_from_sensors_app.py +632 -0
  10. cars/applications/auxiliary_filling/auxiliary_filling_wrappers.py +90 -0
  11. cars/applications/dem_generation/__init__.py +30 -0
  12. cars/applications/dem_generation/abstract_dem_generation_app.py +116 -0
  13. cars/applications/dem_generation/bulldozer_config/base_config.yaml +42 -0
  14. cars/applications/dem_generation/bulldozer_dem_app.py +641 -0
  15. cars/applications/dem_generation/bulldozer_memory.py +55 -0
  16. cars/applications/dem_generation/dem_generation_algo.py +107 -0
  17. cars/applications/dem_generation/dem_generation_constants.py +32 -0
  18. cars/applications/dem_generation/dem_generation_wrappers.py +323 -0
  19. cars/applications/dense_match_filling/__init__.py +30 -0
  20. cars/applications/dense_match_filling/abstract_dense_match_filling_app.py +242 -0
  21. cars/applications/dense_match_filling/fill_disp_algo.py +113 -0
  22. cars/applications/dense_match_filling/fill_disp_constants.py +39 -0
  23. cars/applications/dense_match_filling/fill_disp_wrappers.py +83 -0
  24. cars/applications/dense_match_filling/zero_padding_app.py +302 -0
  25. cars/applications/dense_matching/__init__.py +30 -0
  26. cars/applications/dense_matching/abstract_dense_matching_app.py +261 -0
  27. cars/applications/dense_matching/census_mccnn_sgm_app.py +1461 -0
  28. cars/applications/dense_matching/cpp/__init__.py +0 -0
  29. cars/applications/dense_matching/cpp/dense_matching_cpp.cp312-win_amd64.dll.a +0 -0
  30. cars/applications/dense_matching/cpp/dense_matching_cpp.cp312-win_amd64.pyd +0 -0
  31. cars/applications/dense_matching/cpp/dense_matching_cpp.py +94 -0
  32. cars/applications/dense_matching/cpp/includes/dense_matching.hpp +58 -0
  33. cars/applications/dense_matching/cpp/meson.build +9 -0
  34. cars/applications/dense_matching/cpp/src/bindings.cpp +13 -0
  35. cars/applications/dense_matching/cpp/src/dense_matching.cpp +207 -0
  36. cars/applications/dense_matching/dense_matching_algo.py +401 -0
  37. cars/applications/dense_matching/dense_matching_constants.py +89 -0
  38. cars/applications/dense_matching/dense_matching_wrappers.py +951 -0
  39. cars/applications/dense_matching/disparity_grid_algo.py +597 -0
  40. cars/applications/dense_matching/loaders/__init__.py +23 -0
  41. cars/applications/dense_matching/loaders/config_census_sgm_default.json +31 -0
  42. cars/applications/dense_matching/loaders/config_census_sgm_homogeneous.json +30 -0
  43. cars/applications/dense_matching/loaders/config_census_sgm_mountain_and_vegetation.json +30 -0
  44. cars/applications/dense_matching/loaders/config_census_sgm_shadow.json +30 -0
  45. cars/applications/dense_matching/loaders/config_census_sgm_sparse.json +36 -0
  46. cars/applications/dense_matching/loaders/config_census_sgm_urban.json +30 -0
  47. cars/applications/dense_matching/loaders/config_mapping.json +13 -0
  48. cars/applications/dense_matching/loaders/config_mccnn.json +28 -0
  49. cars/applications/dense_matching/loaders/global_land_cover_map.tif +0 -0
  50. cars/applications/dense_matching/loaders/pandora_loader.py +593 -0
  51. cars/applications/dsm_filling/__init__.py +32 -0
  52. cars/applications/dsm_filling/abstract_dsm_filling_app.py +101 -0
  53. cars/applications/dsm_filling/border_interpolation_app.py +278 -0
  54. cars/applications/dsm_filling/bulldozer_config/base_config.yaml +44 -0
  55. cars/applications/dsm_filling/bulldozer_filling_app.py +288 -0
  56. cars/applications/dsm_filling/exogenous_filling_app.py +341 -0
  57. cars/applications/dsm_merging/__init__.py +28 -0
  58. cars/applications/dsm_merging/abstract_dsm_merging_app.py +101 -0
  59. cars/applications/dsm_merging/weighted_fusion_app.py +639 -0
  60. cars/applications/grid_correction/__init__.py +30 -0
  61. cars/applications/grid_correction/abstract_grid_correction_app.py +103 -0
  62. cars/applications/grid_correction/grid_correction_app.py +557 -0
  63. cars/applications/grid_generation/__init__.py +30 -0
  64. cars/applications/grid_generation/abstract_grid_generation_app.py +142 -0
  65. cars/applications/grid_generation/epipolar_grid_generation_app.py +327 -0
  66. cars/applications/grid_generation/grid_generation_algo.py +388 -0
  67. cars/applications/grid_generation/grid_generation_constants.py +46 -0
  68. cars/applications/grid_generation/transform_grid.py +88 -0
  69. cars/applications/ground_truth_reprojection/__init__.py +30 -0
  70. cars/applications/ground_truth_reprojection/abstract_ground_truth_reprojection_app.py +137 -0
  71. cars/applications/ground_truth_reprojection/direct_localization_app.py +629 -0
  72. cars/applications/ground_truth_reprojection/ground_truth_reprojection_algo.py +275 -0
  73. cars/applications/point_cloud_outlier_removal/__init__.py +30 -0
  74. cars/applications/point_cloud_outlier_removal/abstract_outlier_removal_app.py +385 -0
  75. cars/applications/point_cloud_outlier_removal/outlier_removal_algo.py +392 -0
  76. cars/applications/point_cloud_outlier_removal/outlier_removal_constants.py +43 -0
  77. cars/applications/point_cloud_outlier_removal/small_components_app.py +522 -0
  78. cars/applications/point_cloud_outlier_removal/statistical_app.py +528 -0
  79. cars/applications/rasterization/__init__.py +30 -0
  80. cars/applications/rasterization/abstract_pc_rasterization_app.py +183 -0
  81. cars/applications/rasterization/rasterization_algo.py +534 -0
  82. cars/applications/rasterization/rasterization_constants.py +38 -0
  83. cars/applications/rasterization/rasterization_wrappers.py +639 -0
  84. cars/applications/rasterization/simple_gaussian_app.py +1152 -0
  85. cars/applications/resampling/__init__.py +28 -0
  86. cars/applications/resampling/abstract_resampling_app.py +187 -0
  87. cars/applications/resampling/bicubic_resampling_app.py +760 -0
  88. cars/applications/resampling/resampling_algo.py +590 -0
  89. cars/applications/resampling/resampling_constants.py +36 -0
  90. cars/applications/resampling/resampling_wrappers.py +309 -0
  91. cars/applications/sensors_subsampling/__init__.py +32 -0
  92. cars/applications/sensors_subsampling/abstract_subsampling_app.py +109 -0
  93. cars/applications/sensors_subsampling/rasterio_subsampling_app.py +420 -0
  94. cars/applications/sensors_subsampling/subsampling_algo.py +108 -0
  95. cars/applications/sparse_matching/__init__.py +30 -0
  96. cars/applications/sparse_matching/abstract_sparse_matching_app.py +599 -0
  97. cars/applications/sparse_matching/sift_app.py +724 -0
  98. cars/applications/sparse_matching/sparse_matching_algo.py +360 -0
  99. cars/applications/sparse_matching/sparse_matching_constants.py +66 -0
  100. cars/applications/sparse_matching/sparse_matching_wrappers.py +282 -0
  101. cars/applications/triangulation/__init__.py +32 -0
  102. cars/applications/triangulation/abstract_triangulation_app.py +227 -0
  103. cars/applications/triangulation/line_of_sight_intersection_app.py +1243 -0
  104. cars/applications/triangulation/pc_transform.py +552 -0
  105. cars/applications/triangulation/triangulation_algo.py +371 -0
  106. cars/applications/triangulation/triangulation_constants.py +38 -0
  107. cars/applications/triangulation/triangulation_wrappers.py +259 -0
  108. cars/bundleadjustment.py +750 -0
  109. cars/cars.py +179 -0
  110. cars/conf/__init__.py +23 -0
  111. cars/conf/geoid/egm96.grd +0 -0
  112. cars/conf/geoid/egm96.grd.hdr +15 -0
  113. cars/conf/input_parameters.py +156 -0
  114. cars/conf/mask_cst.py +35 -0
  115. cars/core/__init__.py +23 -0
  116. cars/core/cars_logging.py +402 -0
  117. cars/core/constants.py +191 -0
  118. cars/core/constants_disparity.py +50 -0
  119. cars/core/datasets.py +140 -0
  120. cars/core/geometry/__init__.py +27 -0
  121. cars/core/geometry/abstract_geometry.py +1119 -0
  122. cars/core/geometry/shareloc_geometry.py +598 -0
  123. cars/core/inputs.py +568 -0
  124. cars/core/outputs.py +176 -0
  125. cars/core/preprocessing.py +722 -0
  126. cars/core/projection.py +843 -0
  127. cars/core/roi_tools.py +215 -0
  128. cars/core/tiling.py +774 -0
  129. cars/core/utils.py +164 -0
  130. cars/data_structures/__init__.py +23 -0
  131. cars/data_structures/cars_dataset.py +1544 -0
  132. cars/data_structures/cars_dict.py +74 -0
  133. cars/data_structures/corresponding_tiles_tools.py +186 -0
  134. cars/data_structures/dataframe_converter.py +185 -0
  135. cars/data_structures/format_transformation.py +297 -0
  136. cars/devibrate.py +689 -0
  137. cars/extractroi.py +264 -0
  138. cars/orchestrator/__init__.py +23 -0
  139. cars/orchestrator/achievement_tracker.py +125 -0
  140. cars/orchestrator/cluster/__init__.py +37 -0
  141. cars/orchestrator/cluster/abstract_cluster.py +250 -0
  142. cars/orchestrator/cluster/abstract_dask_cluster.py +381 -0
  143. cars/orchestrator/cluster/dask_cluster_tools.py +103 -0
  144. cars/orchestrator/cluster/dask_config/README.md +94 -0
  145. cars/orchestrator/cluster/dask_config/dask.yaml +21 -0
  146. cars/orchestrator/cluster/dask_config/distributed.yaml +70 -0
  147. cars/orchestrator/cluster/dask_config/jobqueue.yaml +26 -0
  148. cars/orchestrator/cluster/dask_config/reference_confs/dask-schema.yaml +137 -0
  149. cars/orchestrator/cluster/dask_config/reference_confs/dask.yaml +26 -0
  150. cars/orchestrator/cluster/dask_config/reference_confs/distributed-schema.yaml +1009 -0
  151. cars/orchestrator/cluster/dask_config/reference_confs/distributed.yaml +273 -0
  152. cars/orchestrator/cluster/dask_config/reference_confs/jobqueue.yaml +212 -0
  153. cars/orchestrator/cluster/dask_jobqueue_utils.py +204 -0
  154. cars/orchestrator/cluster/local_dask_cluster.py +116 -0
  155. cars/orchestrator/cluster/log_wrapper.py +728 -0
  156. cars/orchestrator/cluster/mp_cluster/__init__.py +27 -0
  157. cars/orchestrator/cluster/mp_cluster/mp_factorizer.py +212 -0
  158. cars/orchestrator/cluster/mp_cluster/mp_objects.py +535 -0
  159. cars/orchestrator/cluster/mp_cluster/mp_tools.py +93 -0
  160. cars/orchestrator/cluster/mp_cluster/mp_wrapper.py +505 -0
  161. cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +986 -0
  162. cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py +399 -0
  163. cars/orchestrator/cluster/pbs_dask_cluster.py +207 -0
  164. cars/orchestrator/cluster/sequential_cluster.py +139 -0
  165. cars/orchestrator/cluster/slurm_dask_cluster.py +234 -0
  166. cars/orchestrator/memory_tools.py +47 -0
  167. cars/orchestrator/orchestrator.py +755 -0
  168. cars/orchestrator/orchestrator_constants.py +29 -0
  169. cars/orchestrator/registry/__init__.py +23 -0
  170. cars/orchestrator/registry/abstract_registry.py +143 -0
  171. cars/orchestrator/registry/compute_registry.py +106 -0
  172. cars/orchestrator/registry/id_generator.py +116 -0
  173. cars/orchestrator/registry/replacer_registry.py +213 -0
  174. cars/orchestrator/registry/saver_registry.py +363 -0
  175. cars/orchestrator/registry/unseen_registry.py +118 -0
  176. cars/orchestrator/tiles_profiler.py +279 -0
  177. cars/pipelines/__init__.py +26 -0
  178. cars/pipelines/conf_resolution/conf_final_resolution.yaml +5 -0
  179. cars/pipelines/conf_resolution/conf_first_resolution.yaml +4 -0
  180. cars/pipelines/conf_resolution/conf_intermediate_resolution.yaml +2 -0
  181. cars/pipelines/default/__init__.py +26 -0
  182. cars/pipelines/default/default_pipeline.py +1088 -0
  183. cars/pipelines/filling/__init__.py +26 -0
  184. cars/pipelines/filling/filling.py +981 -0
  185. cars/pipelines/formatting/__init__.py +26 -0
  186. cars/pipelines/formatting/formatting.py +186 -0
  187. cars/pipelines/merging/__init__.py +26 -0
  188. cars/pipelines/merging/merging.py +439 -0
  189. cars/pipelines/parameters/__init__.py +0 -0
  190. cars/pipelines/parameters/advanced_parameters.py +256 -0
  191. cars/pipelines/parameters/advanced_parameters_constants.py +68 -0
  192. cars/pipelines/parameters/application_parameters.py +72 -0
  193. cars/pipelines/parameters/depth_map_inputs.py +0 -0
  194. cars/pipelines/parameters/dsm_inputs.py +349 -0
  195. cars/pipelines/parameters/dsm_inputs_constants.py +25 -0
  196. cars/pipelines/parameters/output_constants.py +52 -0
  197. cars/pipelines/parameters/output_parameters.py +438 -0
  198. cars/pipelines/parameters/sensor_inputs.py +859 -0
  199. cars/pipelines/parameters/sensor_inputs_constants.py +51 -0
  200. cars/pipelines/parameters/sensor_loaders/__init__.py +29 -0
  201. cars/pipelines/parameters/sensor_loaders/basic_classif_loader.py +86 -0
  202. cars/pipelines/parameters/sensor_loaders/basic_image_loader.py +98 -0
  203. cars/pipelines/parameters/sensor_loaders/pivot_classif_loader.py +90 -0
  204. cars/pipelines/parameters/sensor_loaders/pivot_image_loader.py +105 -0
  205. cars/pipelines/parameters/sensor_loaders/sensor_loader.py +93 -0
  206. cars/pipelines/parameters/sensor_loaders/sensor_loader_template.py +71 -0
  207. cars/pipelines/parameters/sensor_loaders/slurp_classif_loader.py +86 -0
  208. cars/pipelines/pipeline.py +119 -0
  209. cars/pipelines/pipeline_constants.py +38 -0
  210. cars/pipelines/pipeline_template.py +135 -0
  211. cars/pipelines/subsampling/__init__.py +26 -0
  212. cars/pipelines/subsampling/subsampling.py +358 -0
  213. cars/pipelines/surface_modeling/__init__.py +26 -0
  214. cars/pipelines/surface_modeling/surface_modeling.py +2098 -0
  215. cars/pipelines/tie_points/__init__.py +26 -0
  216. cars/pipelines/tie_points/tie_points.py +536 -0
  217. cars/starter.py +167 -0
  218. cars-1.0.0rc2.dist-info/DELVEWHEEL +2 -0
  219. cars-1.0.0rc2.dist-info/METADATA +289 -0
  220. cars-1.0.0rc2.dist-info/RECORD +225 -0
  221. cars-1.0.0rc2.dist-info/WHEEL +4 -0
  222. cars-1.0.0rc2.dist-info/entry_points.txt +8 -0
  223. cars.libs/libgcc_s_seh-1-b2494fcbd4d80cf2c98fdd5261f6d850.dll +0 -0
  224. cars.libs/libstdc++-6-e9b0d12ae0e9555bbae55e8dfd08c3f7.dll +0 -0
  225. cars.libs/libwinpthread-1-7882d1b093714ccdfaf4e0789a817792.dll +0 -0
@@ -0,0 +1,1009 @@
1
+ properties:
2
+ distributed:
3
+ type: object
4
+ properties:
5
+
6
+ version:
7
+ type: integer
8
+
9
+ scheduler:
10
+ type: object
11
+ properties:
12
+
13
+ allowed-failures:
14
+ type: integer
15
+ minimum: 0
16
+ description: |
17
+ The number of retries before a task is considered bad
18
+
19
+ When a worker dies when a task is running that task is rerun elsewhere.
20
+ If many workers die while running this same task then we call the task bad, and raise a KilledWorker exception.
21
+ This is the number of workers that are allowed to die before this task is marked as bad.
22
+
23
+ bandwidth:
24
+ type:
25
+ - integer
26
+ - string
27
+ description: |
28
+ The expected bandwidth between any pair of workers
29
+
30
+ This is used when making scheduling decisions.
31
+ The scheduler will use this value as a baseline, but also learn it over time.
32
+
33
+ blocked-handlers:
34
+ type: array
35
+ description: |
36
+ A list of handlers to exclude
37
+
38
+ The scheduler operates by receiving messages from various workers and clients
39
+ and then performing operations based on those messages.
40
+ Each message has an operation like "close-worker" or "task-finished".
41
+ In some high security situations administrators may choose to block certain handlers
42
+ from running. Those handlers can be listed here.
43
+
44
+ For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
45
+
46
+ default-data-size:
47
+ type:
48
+ - string
49
+ - integer
50
+ description: |
51
+ The default size of a piece of data if we don't know anything about it.
52
+
53
+ This is used by the scheduler in some scheduling decisions
54
+
55
+ events-cleanup-delay:
56
+ type: string
57
+ description: |
58
+ The amount of time to wait until workers or clients are removed from the event log
59
+ after they have been removed from the scheduler
60
+
61
+ idle-timeout:
62
+ type:
63
+ - string
64
+ - "null"
65
+ description: |
66
+ Shut down the scheduler after this duration if no activity has occured
67
+
68
+ This can be helpful to reduce costs and stop zombie processes from roaming the earth.
69
+
70
+ transition-log-length:
71
+ type: integer
72
+ minimum: 0
73
+ description: |
74
+ How long should we keep the transition log
75
+
76
+ Every time a task transitions states (like "waiting", "processing", "memory", "released")
77
+ we record that transition in a log.
78
+
79
+ To make sure that we don't run out of memory
80
+ we will clear out old entries after a certain length.
81
+ This is that length.
82
+
83
+ events-log-length:
84
+ type: integer
85
+ minimum: 0
86
+ description: |
87
+ How long should we keep the events log
88
+
89
+ All events (e.g. worker heartbeat) are stored in the events log.
90
+
91
+ To make sure that we don't run out of memory
92
+ we will clear out old entries after a certain length.
93
+ This is that length.
94
+
95
+ work-stealing:
96
+ type: boolean
97
+ description: |
98
+ Whether or not to balance work between workers dynamically
99
+
100
+ Some times one worker has more work than we expected.
101
+ The scheduler will move these tasks around as necessary by default.
102
+ Set this to false to disable this behavior
103
+
104
+ work-stealing-interval:
105
+ type: string
106
+ description: |
107
+ How frequently to balance worker loads
108
+
109
+ worker-ttl:
110
+ type:
111
+ - string
112
+ - "null"
113
+ description: |
114
+ Time to live for workers.
115
+
116
+ If we don't receive a heartbeat faster than this then we assume that the worker has died.
117
+
118
+ pickle:
119
+ type: boolean
120
+ description: |
121
+ Is the scheduler allowed to deserialize arbitrary bytestrings?
122
+
123
+ The scheduler almost never deserializes user data.
124
+ However there are some cases where the user can submit functions to run directly on the scheduler.
125
+ This can be convenient for debugging, but also introduces some security risk.
126
+ By setting this to false we ensure that the user is unable to run arbitrary code on the scheduler.
127
+
128
+ preload:
129
+ type: array
130
+ description: |
131
+ Run custom modules during the lifetime of the scheduler
132
+
133
+ You can run custom modules when the scheduler starts up and closes down.
134
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
135
+
136
+ preload-argv:
137
+ type: array
138
+ description: |
139
+ Arguments to pass into the preload scripts described above
140
+
141
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
142
+
143
+ unknown-task-duration:
144
+ type: string
145
+ description: |
146
+ Default duration for all tasks with unknown durations
147
+
148
+ Over time the scheduler learns a duration for tasks.
149
+ However when it sees a new type of task for the first time it has to make a guess
150
+ as to how long it will take. This value is that guess.
151
+
152
+ default-task-durations:
153
+ type: object
154
+ description: |
155
+ How long we expect function names to run
156
+
157
+ Over time the scheduler will learn these values, but these give it a good starting point.
158
+
159
+ validate:
160
+ type: boolean
161
+ description: |
162
+ Whether or not to run consistency checks during execution.
163
+ This is typically only used for debugging.
164
+
165
+ dashboard:
166
+ type: object
167
+ description: |
168
+ Configuration options for Dask's real-time dashboard
169
+
170
+ properties:
171
+ status:
172
+ type: object
173
+ description: The main status page of the dashboard
174
+ properties:
175
+ task-stream-length:
176
+ type: integer
177
+ minimum: 0
178
+ description: |
179
+ The maximum number of tasks to include in the task stream plot
180
+ tasks:
181
+ type: object
182
+ description: |
183
+ The page which includes the full task stream history
184
+ properties:
185
+ task-stream-length:
186
+ type: integer
187
+ minimum: 0
188
+ description: |
189
+ The maximum number of tasks to include in the task stream plot
190
+ tls:
191
+ type: object
192
+ description: |
193
+ Settings around securing the dashboard
194
+ properties:
195
+ ca-file:
196
+ type:
197
+ - string
198
+ - "null"
199
+ key:
200
+ type:
201
+ - string
202
+ - "null"
203
+ cert:
204
+ type:
205
+ - string
206
+ - "null"
207
+ bokeh-application:
208
+ type: object
209
+ description: |
210
+ Keywords to pass to the BokehTornado application
211
+ locks:
212
+ type: object
213
+ description: |
214
+ Settings for Dask's distributed Lock object
215
+
216
+ See https://docs.dask.org/en/latest/futures.html#locks for more information
217
+ properties:
218
+ lease-validation-interval:
219
+ type: string
220
+ description: |
221
+ The interval in which the scheduler validates staleness of all acquired leases. Must always be smaller than the lease-timeout itself.
222
+ lease-timeout:
223
+ type: string
224
+ description: |
225
+ Maximum interval to wait for a Client refresh before a lease is invalidated and released.
226
+
227
+ http:
228
+ type: object
229
+ decription: Settings for Dask's embedded HTTP Server
230
+ properties:
231
+ routes:
232
+ type: array
233
+ description: |
234
+ A list of modules like "prometheus" and "health" that can be included or excluded as desired
235
+
236
+ These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
237
+ This is also a list that can be extended with user defined modules.
238
+
239
+ allowed-imports:
240
+ type: array
241
+ description: |
242
+ A list of trusted root modules the schedular is allowed to import (incl. submodules). For security reasons, the
243
+ scheduler does not import arbitrary Python modules.
244
+
245
+ active-memory-manager:
246
+ type: object
247
+ required: [start, interval, policies]
248
+ additionalProperties: false
249
+ properties:
250
+ start:
251
+ type: boolean
252
+ description: set to true to auto-start the AMM on Scheduler init
253
+ interval:
254
+ type: string
255
+ description:
256
+ Time expression, e.g. "2s". Run the AMM cycle every <interval>.
257
+ policies:
258
+ type: array
259
+ items:
260
+ type: object
261
+ required: [class]
262
+ properties:
263
+ class:
264
+ type: string
265
+ description: fully qualified name of an ActiveMemoryManagerPolicy
266
+ subclass
267
+ additionalProperties:
268
+ description: keyword arguments to the policy constructor, if any
269
+
270
+ worker:
271
+ type: object
272
+ description: |
273
+ Configuration settings for Dask Workers
274
+ properties:
275
+ blocked-handlers:
276
+ type: array
277
+ description: |
278
+ A list of handlers to exclude
279
+
280
+ The scheduler operates by receiving messages from various workers and clients
281
+ and then performing operations based on those messages.
282
+ Each message has an operation like "close-worker" or "task-finished".
283
+ In some high security situations administrators may choose to block certain handlers
284
+ from running. Those handlers can be listed here.
285
+
286
+ For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
287
+
288
+ multiprocessing-method:
289
+ type: string
290
+ description: |
291
+ How we create new workers, one of "spawn", "forkserver", or "fork"
292
+
293
+ This is passed to the ``multiprocessing.get_context`` function.
294
+ use-file-locking:
295
+ type: boolean
296
+ description: |
297
+ Whether or not to use lock files when creating workers
298
+
299
+ Workers create a local directory in which to place temporary files.
300
+ When many workers are created on the same process at once
301
+ these workers can conflict with each other by trying to create this directory all at the same time.
302
+
303
+ To avoid this, Dask usually used a file-based lock.
304
+ However, on some systems file-based locks don't work.
305
+ This is particularly common on HPC NFS systems, where users may want to set this to false.
306
+ connections:
307
+ type: object
308
+ description: |
309
+ The number of concurrent connections to allow to other workers
310
+ properties:
311
+ incoming:
312
+ type: integer
313
+ minimum: 0
314
+ outgoing:
315
+ type: integer
316
+ minimum: 0
317
+
318
+ preload:
319
+ type: array
320
+ description: |
321
+ Run custom modules during the lifetime of the worker
322
+
323
+ You can run custom modules when the worker starts up and closes down.
324
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
325
+
326
+ preload-argv:
327
+ type: array
328
+ description: |
329
+ Arguments to pass into the preload scripts described above
330
+
331
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
332
+
333
+ daemon:
334
+ type: boolean
335
+ description: |
336
+ Whether or not to run our process as a daemon process
337
+
338
+ validate:
339
+ type: boolean
340
+ description: |
341
+ Whether or not to run consistency checks during execution.
342
+ This is typically only used for debugging.
343
+
344
+ resources:
345
+ type: object
346
+ description: |
347
+ A dictionary specifying resources for workers.
348
+
349
+ See https://distributed.dask.org/en/latest/resources.html for more information.
350
+ properties: {}
351
+
352
+ lifetime:
353
+ type: object
354
+ description: |
355
+ The worker may choose to gracefully close itself down after some pre-determined time.
356
+
357
+ This is particularly useful if you know that your worker job has a time limit on it.
358
+ This is particularly common in HPC job schedulers.
359
+
360
+ For example if your worker has a walltime of one hour,
361
+ then you may want to set the lifetime.duration to "55 minutes"
362
+ properties:
363
+ duration:
364
+ type:
365
+ - string
366
+ - "null"
367
+ description: |
368
+ The time after creation to close the worker, like "1 hour"
369
+ stagger:
370
+ type: string
371
+ description: |
372
+ Random amount by which to stagger lifetimes
373
+
374
+ If you create many workers at the same time,
375
+ you may want to avoid having them kill themselves all at the same time.
376
+ To avoid this you might want to set a stagger time,
377
+ so that they close themselves with some random variation, like "5 minutes"
378
+
379
+ That way some workers can die, new ones can be brought up,
380
+ and data can be transferred over smoothly.
381
+ restart:
382
+ type: boolean
383
+ description: |
384
+ Do we try to resurrect the worker after the lifetime deadline?
385
+
386
+
387
+ profile:
388
+ type: object
389
+ description: |
390
+ The workers periodically poll every worker thread to see what they are working on.
391
+ This data gets collected into statistical profiling information,
392
+ which is then periodically bundled together and sent along to the scheduler.
393
+ properties:
394
+ interval:
395
+ type: string
396
+ description: |
397
+ The time between polling the worker threads, typically short like 10ms
398
+ cycle:
399
+ type: string
400
+ description: |
401
+ The time between bundling together this data and sending it to the scheduler
402
+
403
+ This controls the granularity at which people can query the profile information
404
+ on the time axis.
405
+ low-level:
406
+ type: boolean
407
+ description: |
408
+ Whether or not to use the libunwind and stacktrace libraries
409
+ to gather profiling information at the lower level (beneath Python)
410
+
411
+ To get this to work you will need to install the experimental stacktrace library at
412
+
413
+ conda install -c numba stacktrace
414
+
415
+ See https://github.com/numba/stacktrace
416
+
417
+ memory:
418
+ type: object
419
+ description: >-
420
+ Settings for memory management
421
+ properties:
422
+ recent-to-old-time:
423
+ type: string
424
+ description: >-
425
+ When there is an increase in process memory (as observed by the
426
+ operating system) that is not accounted for by the dask keys stored on
427
+ the worker, ignore it for this long before considering it in
428
+ non-time-sensitive heuristics. This should be set to be longer than
429
+ the duration of most dask tasks.
430
+ rebalance:
431
+ type: object
432
+ description: >-
433
+ Settings for memory rebalance operations
434
+ properties:
435
+ measure:
436
+ enum:
437
+ - process
438
+ - optimistic
439
+ - managed
440
+ - managed_in_memory
441
+ description: >-
442
+ Which of the properties of distributed.scheduler.MemoryState
443
+ should be used for measuring worker memory usage
444
+ sender-min:
445
+ type: number
446
+ minimum: 0
447
+ maximum: 1
448
+ description: >-
449
+ Fraction of worker process memory at which we start potentially
450
+ transferring data to other workers.
451
+ recipient-max:
452
+ type: number
453
+ minimum: 0
454
+ maximum: 1
455
+ description: >-
456
+ Fraction of worker process memory at which we stop potentially
457
+ receiving data from other workers. Ignored when max_memory is not
458
+ set.
459
+ sender-recipient-gap:
460
+ type: number
461
+ minimum: 0
462
+ maximum: 1
463
+ description: >-
464
+ Fraction of worker process memory, around the cluster mean, where
465
+ a worker is neither a sender nor a recipient of data during a
466
+ rebalance operation. E.g. if the mean cluster occupation is 50%,
467
+ sender-recipient-gap=0.1 means that only nodes above 55% will
468
+ donate data and only nodes below 45% will receive them. This helps
469
+ avoid data from bouncing around the cluster repeatedly.
470
+
471
+ target:
472
+ oneOf:
473
+ - {type: number, minimum: 0, maximum: 1}
474
+ - {enum: [false]}
475
+ description: >-
476
+ When the process memory (as observed by the operating system) gets
477
+ above this amount we start spilling the dask keys holding the largest
478
+ chunks of data to disk
479
+
480
+ spill:
481
+ oneOf:
482
+ - {type: number, minimum: 0, maximum: 1}
483
+ - {enum: [false]}
484
+ description: >-
485
+ When the process memory (as observed by the operating system) gets
486
+ above this amount we spill all data to disk.
487
+
488
+ pause:
489
+ oneOf:
490
+ - {type: number, minimum: 0, maximum: 1}
491
+ - {enum: [false]}
492
+ description: >-
493
+ When the process memory (as observed by the operating system) gets
494
+ above this amount we no longer start new tasks on this worker.
495
+
496
+ terminate:
497
+ oneOf:
498
+ - {type: number, minimum: 0, maximum: 1}
499
+ - {enum: [false]}
500
+ description: >-
501
+ When the process memory reaches this level the nanny process will kill
502
+ the worker (if a nanny is present)
503
+
504
+ http:
505
+ type: object
506
+ decription: Settings for Dask's embedded HTTP Server
507
+ properties:
508
+ routes:
509
+ type: array
510
+ description: |
511
+ A list of modules like "prometheus" and "health" that can be included or excluded as desired
512
+
513
+ These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
514
+ This is also a list that can be extended with user defined modules.
515
+
516
+ nanny:
517
+ type: object
518
+ description: |
519
+ Configuration settings for Dask Nannies
520
+ properties:
521
+
522
+ preload:
523
+ type: array
524
+ description: |
525
+ Run custom modules during the lifetime of the scheduler
526
+
527
+ You can run custom modules when the scheduler starts up and closes down.
528
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
529
+
530
+ preload-argv:
531
+ type: array
532
+ description: |
533
+ Arguments to pass into the preload scripts described above
534
+
535
+ See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
536
+
537
+ environ:
538
+ type: object
539
+ description: |
540
+ Environment variables to set on all worker processes started by nannies
541
+
542
+ client:
543
+ type: object
544
+ description: |
545
+ Configuration settings for Dask Clients
546
+
547
+ properties:
548
+ heartbeat:
549
+ type: string
550
+ description:
551
+ This value is the time between heartbeats
552
+
553
+ The client sends a periodic heartbeat message to the scheduler.
554
+ If it misses enough of these then the scheduler assumes that it has gone.
555
+
556
+ scheduler-info-interval:
557
+ type: string
558
+ description: Interval between scheduler-info updates
559
+
560
+ deploy:
561
+ type: object
562
+ description: Configuration settings for general Dask deployment
563
+ properties:
564
+ lost-worker-timeout:
565
+ type: string
566
+ description: |
567
+ Interval after which to hard-close a lost worker job
568
+
569
+ Otherwise we wait for a while to see if a worker will reappear
570
+
571
+ cluster-repr-interval:
572
+ type: string
573
+ description: Interval between calls to update cluster-repr for the widget
574
+
575
+ adaptive:
576
+ type: object
577
+ description: Configuration settings for Dask's adaptive scheduling
578
+ properties:
579
+ interval:
580
+ type: string
581
+ description: |
582
+ The duration between checking in with adaptive scheduling load
583
+
584
+ The adaptive system periodically checks scheduler load and determines
585
+ if it should scale the cluster up or down.
586
+ This is the timing between those checks.
587
+
588
+ target-duration:
589
+ type: string
590
+ description: |
591
+ The desired time for the entire computation to run
592
+
593
+ The adaptive system will try to start up enough workers to run
594
+ the computation in about this time.
595
+
596
+ minimum:
597
+ type: integer
598
+ minimum: 0
599
+ description: |
600
+ The minimum number of workers to keep around
601
+
602
+ maximum:
603
+ type: number
604
+ minimum: 0
605
+ description: |
606
+ The maximum number of workers to keep around
607
+
608
+ wait-count:
609
+ type: integer
610
+ minimum: 1
611
+ description: |
612
+ The number of times a worker should be suggested for removal before removing it
613
+
614
+ This helps to smooth out the number of deployed workers
615
+
616
+ comm:
617
+ type: object
618
+ description: Configuration settings for Dask communications
619
+ properties:
620
+
621
+ retry:
622
+ type: object
623
+ description: |
624
+ Some operations (such as gathering data) are subject to re-tries with the below parameters
625
+ properties:
626
+
627
+ count:
628
+ type: integer
629
+ minimum: 0
630
+ description: |
631
+ The number of times to retry a connection
632
+
633
+ delay:
634
+ type: object
635
+ properties:
636
+ min:
637
+ type: string
638
+ description: The first non-zero delay between retry attempts
639
+ max:
640
+ type: string
641
+ description: The maximum delay between retries
642
+
643
+ compression:
644
+ type: string
645
+ description: |
646
+ The compression algorithm to use
647
+
648
+ This could be one of lz4, snappy, zstd, or blosc
649
+
650
+ offload:
651
+ type:
652
+ - boolean
653
+ - string
654
+ description: |
655
+ The size of message after which we choose to offload serialization to another thread
656
+
657
+ In some cases, you may also choose to disable this altogether with the value false
658
+ This is useful if you want to include serialization in profiling data,
659
+ or if you have data types that are particularly sensitive to deserialization
660
+
661
+ shard:
662
+ type: string
663
+ description: |
664
+ The maximum size of a frame to send through a comm
665
+
666
+ Some network infrastructure doesn't like sending through very large messages.
667
+ Dask comms will cut up these large messages into many small ones.
668
+ This attribute determines the maximum size of such a shard.
669
+
670
+ socket-backlog:
671
+ type: integer
672
+ description: |
673
+ When shuffling data between workers, there can
674
+ really be O(cluster size) connection requests
675
+ on a single worker socket, make sure the backlog
676
+ is large enough not to lose any.
677
+
678
+ zstd:
679
+ type: object
680
+ description: Options for the Z Standard compression scheme
681
+ properties:
682
+ level:
683
+ type: integer
684
+ minimum: 1
685
+ maximum: 22
686
+ description: Compression level, between 1 and 22.
687
+ threads:
688
+ type: integer
689
+ minimum: -1
690
+ description: |
691
+ Number of threads to use.
692
+
693
+ 0 for single-threaded, -1 to infer from cpu count.
694
+
695
+ timeouts:
696
+ type: object
697
+ properties:
698
+ connect:
699
+ type: string
700
+ tcp:
701
+ type: string
702
+
703
+ require-encryption:
704
+ type:
705
+ - boolean
706
+ - "null"
707
+ description: |
708
+ Whether to require encryption on non-local comms
709
+
710
+ default-scheme:
711
+ type: string
712
+ description: The default protocol to use, like tcp or tls
713
+
714
+ recent-messages-log-length:
715
+ type: integer
716
+ minimum: 0
717
+ description: number of messages to keep for debugging
718
+
719
+ tls:
720
+ type: object
721
+ properties:
722
+ ciphers:
723
+ type:
724
+ - string
725
+ - "null"
726
+ description: Allowed ciphers, specified as an OpenSSL cipher string.
727
+
728
+ min-version:
729
+ enum: [null, 1.2, 1.3]
730
+ description: The minimum TLS version to support. Defaults to TLS 1.2.
731
+
732
+ max-version:
733
+ enum: [null, 1.2, 1.3]
734
+ description: |
735
+ The maximum TLS version to support. Defaults to the maximum
736
+ version supported by the platform.
737
+
738
+ ca-file:
739
+ type:
740
+ - string
741
+ - "null"
742
+ description: Path to a CA file, in pem format
743
+
744
+ scheduler:
745
+ type: object
746
+ description: TLS information for the scheduler
747
+ properties:
748
+ cert:
749
+ type:
750
+ - string
751
+ - "null"
752
+ description: Path to certificate file
753
+ key:
754
+ type:
755
+ - string
756
+ - "null"
757
+ description: |
758
+ Path to key file.
759
+
760
+ Alternatively, the key can be appended to the cert file
761
+ above, and this field left blank
762
+
763
+ worker:
764
+ type: object
765
+ description: TLS information for the worker
766
+ properties:
767
+ cert:
768
+ type:
769
+ - string
770
+ - "null"
771
+ description: Path to certificate file
772
+ key:
773
+ type:
774
+ - string
775
+ - "null"
776
+ description: |
777
+ Path to key file.
778
+
779
+ Alternatively, the key can be appended to the cert file
780
+ above, and this field left blank
781
+
782
+ client:
783
+ type: object
784
+ description: TLS information for the client
785
+ properties:
786
+ cert:
787
+ type:
788
+ - string
789
+ - "null"
790
+ description: Path to certificate file
791
+ key:
792
+ type:
793
+ - string
794
+ - "null"
795
+ description: |
796
+ Path to key file.
797
+
798
+ Alternatively, the key can be appended to the cert file
799
+ above, and this field left blank
800
+
801
+ ucx:
802
+ type: object
803
+ description: |
804
+ UCX provides access to other transport methods including NVLink and InfiniBand.
805
+ properties:
806
+ cuda-copy:
807
+ type: [boolean, 'null']
808
+ description: |
809
+ Set environment variables to enable CUDA support over UCX. This may be used even if
810
+ InfiniBand and NVLink are not supported or disabled, then transferring data over TCP.
811
+ tcp:
812
+ type: [boolean, 'null']
813
+ description: |
814
+ Set environment variables to enable TCP over UCX, even if InfiniBand and NVLink
815
+ are not supported or disabled.
816
+ nvlink:
817
+ type: [boolean, 'null']
818
+ description: |
819
+ Set environment variables to enable UCX over NVLink, implies ``distributed.comm.ucx.tcp=True``.
820
+ infiniband:
821
+ type: [boolean, 'null']
822
+ description: |
823
+ Set environment variables to enable UCX over InfiniBand, implies ``distributed.comm.ucx.tcp=True``.
824
+ rdmacm:
825
+ type: [boolean, 'null']
826
+ description: |
827
+ Set environment variables to enable UCX RDMA connection manager support,
828
+ requires ``distributed.comm.ucx.infiniband=True``.
829
+ net-devices:
830
+ type: [string, 'null']
831
+ description: |
832
+ Interface(s) used by workers for UCX communication. Can be a string (like
833
+ ``"eth0"`` for NVLink or ``"mlx5_0:1"``/``"ib0"`` for InfiniBand), ``"auto"``
834
+ (requires ``distributed.comm.ucx.infiniband=True``) to pick the optimal interface per-worker based on
835
+ the system's topology, or ``None`` to stay with the default value of ``"all"`` (use
836
+ all available interfaces). Setting to ``"auto"`` requires UCX-Py to be installed
837
+ and compiled with hwloc support. Unexpected errors can occur when using
838
+ ``"auto"`` if any interfaces are disconnected or improperly configured.
839
+ reuse-endpoints:
840
+ type: [boolean, 'null']
841
+ description: |
842
+ Enable UCX-Py reuse endpoints mechanism if ``True`` or if it's not specified and
843
+ UCX < 1.11 is installed, otherwise disable reuse endpoints. This was primarily
844
+ introduced to resolve an issue with CUDA IPC that has been fixed in UCX 1.10, but
845
+ can cause establishing endpoints to be very slow, this is particularly noticeable in
846
+ clusters of more than a few dozen workers.
847
+ create-cuda-context:
848
+ type: [boolean, 'null']
849
+ description: |
850
+ Creates a CUDA context before UCX is initialized. This is necessary to enable UCX to
851
+ properly identify connectivity of GPUs with specialized networking hardware, such as
852
+ InfiniBand. This permits UCX to choose transports automatically, without specifying
853
+ additional variables for each transport, while ensuring optimal connectivity. When
854
+ ``True``, a CUDA context will be created on the first device listed in
855
+ ``CUDA_VISIBLE_DEVICES``.
856
+
857
+ tcp:
858
+ type: object
859
+ properties:
860
+ backend:
861
+ type: string
862
+ description: |
863
+ The TCP backend implementation to use. Must be either `tornado` or `asyncio`.
864
+
865
+ websockets:
866
+ type: object
867
+ properties:
868
+ shard:
869
+ type:
870
+ - string
871
+ description: |
872
+ The maximum size of a websocket frame to send through a comm.
873
+
874
+ This is somewhat duplicative of distributed.comm.shard, but websockets
875
+ often have much smaller maximum message sizes than othe protocols, so
876
+ this attribute is used to set a smaller default shard size and to
877
+ allow separate control of websocket message sharding.
878
+
879
+ diagnostics:
880
+ type: object
881
+ properties:
882
+ nvml:
883
+ type: boolean
884
+ description: |
885
+ If ``True``, enables GPU diagnostics with NVML. Generally leaving it enabled is
886
+ not a problem and will be automatically disabled if no GPUs are found in the
887
+ system, but in certain cases it may be desirable to completely disable NVML
888
+ diagnostics.
889
+ computations:
890
+ type: object
891
+ properties:
892
+ max-history:
893
+ type: integer
894
+ minimum: 0
895
+ description: |
896
+ The maximum number of Computations to remember.
897
+ ignore-modules:
898
+ type: array
899
+ description: |
900
+ A list of modules which are ignored when trying to collect the
901
+ code context when submitting a computation. Accepts regular
902
+ expressions.
903
+
904
+ dashboard:
905
+ type: object
906
+ properties:
907
+ link:
908
+ type: string
909
+ description: |
910
+ The form for the dashboard links
911
+
912
+ This is used wherever we print out the link for the dashboard
913
+ It is filled in with relevant information like the schema, host, and port number
914
+ graph-max-items:
915
+ type: integer
916
+ minimum: 0
917
+ description: maximum number of tasks to try to plot in "graph" view
918
+
919
+ export-tool:
920
+ type: boolean
921
+
922
+ prometheus:
923
+ type: object
924
+ properties:
925
+ namespace:
926
+ type: string
927
+ description: Namespace prefix to use for all prometheus metrics.
928
+
929
+ admin:
930
+ type: object
931
+ description: |
932
+ Options for logs, event loops, and so on
933
+ properties:
934
+ tick:
935
+ type: object
936
+ description: |
937
+ Time between event loop health checks
938
+
939
+ We set up a periodic callback to run on the event loop and check in fairly frequently.
940
+ (by default, this is every 20 milliseconds)
941
+
942
+ If this periodic callback sees that the last time it checked in was several seconds ago
943
+ (by default, this is 3 seconds)
944
+ then it logs a warning saying that something has been stopping the event loop from smooth operation.
945
+ This is typically caused by GIL holding operations,
946
+ but could also be several other things.
947
+
948
+ properties:
949
+ interval:
950
+ type: string
951
+ description: The time between ticks, default 20ms
952
+ limit :
953
+ type: string
954
+ description: The time allowed before triggering a warning
955
+
956
+ max-error-length:
957
+ type: integer
958
+ minimum: 0
959
+ description: |
960
+ Maximum length of traceback as text
961
+
962
+ Some Python tracebacks can be very very long
963
+ (particularly in stack overflow errors)
964
+
965
+ If the traceback is larger than this size (in bytes) then we truncate it.
966
+
967
+ log-length:
968
+ type: integer
969
+ minimum: 0
970
+ description: |
971
+ Default length of logs to keep in memory
972
+
973
+ The scheduler and workers keep the last 10000 or so log entries in memory.
974
+
975
+ log-format:
976
+ type: string
977
+ description: |
978
+ The log format to emit.
979
+
980
+ See https://docs.python.org/3/library/logging.html#logrecord-attributes
981
+ event-loop:
982
+ type: string
983
+ description: |
984
+ The event loop to use,
985
+
986
+ Must be one of tornado, asyncio, or uvloop
987
+
988
+ pdb-on-err:
989
+ type: boolean
990
+ description: Enter Python Debugger on scheduling error
991
+
992
+ system-monitor:
993
+ type: object
994
+ description: |
995
+ Options for the periodic system monitor
996
+ properties:
997
+ interval:
998
+ type: string
999
+ description: Polling time to query cpu/memory statistics default 500ms
1000
+
1001
+ rmm:
1002
+ type: object
1003
+ description: |
1004
+ Configuration options for the RAPIDS Memory Manager.
1005
+ properties:
1006
+ pool-size:
1007
+ type: [integer, 'null']
1008
+ description: |
1009
+ The size of the memory pool in bytes.