cars 1.0.0rc1__cp313-cp313-musllinux_1_2_i686.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cars might be problematic. Click here for more details.
- cars/__init__.py +74 -0
- cars/applications/__init__.py +37 -0
- cars/applications/application.py +117 -0
- cars/applications/application_constants.py +29 -0
- cars/applications/application_template.py +146 -0
- cars/applications/auxiliary_filling/__init__.py +29 -0
- cars/applications/auxiliary_filling/abstract_auxiliary_filling_app.py +104 -0
- cars/applications/auxiliary_filling/auxiliary_filling_algo.py +475 -0
- cars/applications/auxiliary_filling/auxiliary_filling_from_sensors_app.py +630 -0
- cars/applications/auxiliary_filling/auxiliary_filling_wrappers.py +90 -0
- cars/applications/dem_generation/__init__.py +30 -0
- cars/applications/dem_generation/abstract_dem_generation_app.py +116 -0
- cars/applications/dem_generation/bulldozer_config/base_config.yaml +42 -0
- cars/applications/dem_generation/bulldozer_dem_app.py +655 -0
- cars/applications/dem_generation/bulldozer_memory.py +55 -0
- cars/applications/dem_generation/dem_generation_algo.py +107 -0
- cars/applications/dem_generation/dem_generation_constants.py +32 -0
- cars/applications/dem_generation/dem_generation_wrappers.py +323 -0
- cars/applications/dense_match_filling/__init__.py +30 -0
- cars/applications/dense_match_filling/abstract_dense_match_filling_app.py +242 -0
- cars/applications/dense_match_filling/fill_disp_algo.py +113 -0
- cars/applications/dense_match_filling/fill_disp_constants.py +39 -0
- cars/applications/dense_match_filling/fill_disp_wrappers.py +83 -0
- cars/applications/dense_match_filling/zero_padding_app.py +302 -0
- cars/applications/dense_matching/__init__.py +30 -0
- cars/applications/dense_matching/abstract_dense_matching_app.py +261 -0
- cars/applications/dense_matching/census_mccnn_sgm_app.py +1460 -0
- cars/applications/dense_matching/cpp/__init__.py +0 -0
- cars/applications/dense_matching/cpp/dense_matching_cpp.cpython-313-i386-linux-musl.so +0 -0
- cars/applications/dense_matching/cpp/dense_matching_cpp.py +94 -0
- cars/applications/dense_matching/cpp/includes/dense_matching.hpp +58 -0
- cars/applications/dense_matching/cpp/meson.build +9 -0
- cars/applications/dense_matching/cpp/src/bindings.cpp +13 -0
- cars/applications/dense_matching/cpp/src/dense_matching.cpp +207 -0
- cars/applications/dense_matching/dense_matching_algo.py +401 -0
- cars/applications/dense_matching/dense_matching_constants.py +89 -0
- cars/applications/dense_matching/dense_matching_wrappers.py +951 -0
- cars/applications/dense_matching/disparity_grid_algo.py +588 -0
- cars/applications/dense_matching/loaders/__init__.py +23 -0
- cars/applications/dense_matching/loaders/config_census_sgm_default.json +31 -0
- cars/applications/dense_matching/loaders/config_census_sgm_homogeneous.json +30 -0
- cars/applications/dense_matching/loaders/config_census_sgm_mountain_and_vegetation.json +30 -0
- cars/applications/dense_matching/loaders/config_census_sgm_shadow.json +30 -0
- cars/applications/dense_matching/loaders/config_census_sgm_sparse.json +36 -0
- cars/applications/dense_matching/loaders/config_census_sgm_urban.json +30 -0
- cars/applications/dense_matching/loaders/config_mapping.json +13 -0
- cars/applications/dense_matching/loaders/config_mccnn.json +28 -0
- cars/applications/dense_matching/loaders/global_land_cover_map.tif +0 -0
- cars/applications/dense_matching/loaders/pandora_loader.py +593 -0
- cars/applications/dsm_filling/__init__.py +32 -0
- cars/applications/dsm_filling/abstract_dsm_filling_app.py +101 -0
- cars/applications/dsm_filling/border_interpolation_app.py +270 -0
- cars/applications/dsm_filling/bulldozer_config/base_config.yaml +44 -0
- cars/applications/dsm_filling/bulldozer_filling_app.py +279 -0
- cars/applications/dsm_filling/exogenous_filling_app.py +333 -0
- cars/applications/grid_generation/__init__.py +30 -0
- cars/applications/grid_generation/abstract_grid_generation_app.py +142 -0
- cars/applications/grid_generation/epipolar_grid_generation_app.py +327 -0
- cars/applications/grid_generation/grid_correction_app.py +496 -0
- cars/applications/grid_generation/grid_generation_algo.py +388 -0
- cars/applications/grid_generation/grid_generation_constants.py +46 -0
- cars/applications/grid_generation/transform_grid.py +88 -0
- cars/applications/ground_truth_reprojection/__init__.py +30 -0
- cars/applications/ground_truth_reprojection/abstract_ground_truth_reprojection_app.py +137 -0
- cars/applications/ground_truth_reprojection/direct_localization_app.py +629 -0
- cars/applications/ground_truth_reprojection/ground_truth_reprojection_algo.py +275 -0
- cars/applications/point_cloud_outlier_removal/__init__.py +30 -0
- cars/applications/point_cloud_outlier_removal/abstract_outlier_removal_app.py +385 -0
- cars/applications/point_cloud_outlier_removal/outlier_removal_algo.py +392 -0
- cars/applications/point_cloud_outlier_removal/outlier_removal_constants.py +43 -0
- cars/applications/point_cloud_outlier_removal/small_components_app.py +527 -0
- cars/applications/point_cloud_outlier_removal/statistical_app.py +531 -0
- cars/applications/rasterization/__init__.py +30 -0
- cars/applications/rasterization/abstract_pc_rasterization_app.py +183 -0
- cars/applications/rasterization/rasterization_algo.py +534 -0
- cars/applications/rasterization/rasterization_constants.py +38 -0
- cars/applications/rasterization/rasterization_wrappers.py +634 -0
- cars/applications/rasterization/simple_gaussian_app.py +1152 -0
- cars/applications/resampling/__init__.py +28 -0
- cars/applications/resampling/abstract_resampling_app.py +187 -0
- cars/applications/resampling/bicubic_resampling_app.py +762 -0
- cars/applications/resampling/resampling_algo.py +614 -0
- cars/applications/resampling/resampling_constants.py +36 -0
- cars/applications/resampling/resampling_wrappers.py +309 -0
- cars/applications/sparse_matching/__init__.py +30 -0
- cars/applications/sparse_matching/abstract_sparse_matching_app.py +498 -0
- cars/applications/sparse_matching/sift_app.py +735 -0
- cars/applications/sparse_matching/sparse_matching_algo.py +360 -0
- cars/applications/sparse_matching/sparse_matching_constants.py +68 -0
- cars/applications/sparse_matching/sparse_matching_wrappers.py +238 -0
- cars/applications/triangulation/__init__.py +32 -0
- cars/applications/triangulation/abstract_triangulation_app.py +227 -0
- cars/applications/triangulation/line_of_sight_intersection_app.py +1243 -0
- cars/applications/triangulation/pc_transform.py +552 -0
- cars/applications/triangulation/triangulation_algo.py +371 -0
- cars/applications/triangulation/triangulation_constants.py +38 -0
- cars/applications/triangulation/triangulation_wrappers.py +259 -0
- cars/bundleadjustment.py +757 -0
- cars/cars.py +177 -0
- cars/conf/__init__.py +23 -0
- cars/conf/geoid/egm96.grd +0 -0
- cars/conf/geoid/egm96.grd.hdr +15 -0
- cars/conf/input_parameters.py +156 -0
- cars/conf/mask_cst.py +35 -0
- cars/core/__init__.py +23 -0
- cars/core/cars_logging.py +402 -0
- cars/core/constants.py +191 -0
- cars/core/constants_disparity.py +50 -0
- cars/core/datasets.py +140 -0
- cars/core/geometry/__init__.py +27 -0
- cars/core/geometry/abstract_geometry.py +1119 -0
- cars/core/geometry/shareloc_geometry.py +598 -0
- cars/core/inputs.py +568 -0
- cars/core/outputs.py +176 -0
- cars/core/preprocessing.py +722 -0
- cars/core/projection.py +843 -0
- cars/core/roi_tools.py +215 -0
- cars/core/tiling.py +774 -0
- cars/core/utils.py +164 -0
- cars/data_structures/__init__.py +23 -0
- cars/data_structures/cars_dataset.py +1541 -0
- cars/data_structures/cars_dict.py +74 -0
- cars/data_structures/corresponding_tiles_tools.py +186 -0
- cars/data_structures/dataframe_converter.py +185 -0
- cars/data_structures/format_transformation.py +297 -0
- cars/devibrate.py +689 -0
- cars/extractroi.py +264 -0
- cars/orchestrator/__init__.py +23 -0
- cars/orchestrator/achievement_tracker.py +125 -0
- cars/orchestrator/cluster/__init__.py +37 -0
- cars/orchestrator/cluster/abstract_cluster.py +244 -0
- cars/orchestrator/cluster/abstract_dask_cluster.py +375 -0
- cars/orchestrator/cluster/dask_cluster_tools.py +103 -0
- cars/orchestrator/cluster/dask_config/README.md +94 -0
- cars/orchestrator/cluster/dask_config/dask.yaml +21 -0
- cars/orchestrator/cluster/dask_config/distributed.yaml +70 -0
- cars/orchestrator/cluster/dask_config/jobqueue.yaml +26 -0
- cars/orchestrator/cluster/dask_config/reference_confs/dask-schema.yaml +137 -0
- cars/orchestrator/cluster/dask_config/reference_confs/dask.yaml +26 -0
- cars/orchestrator/cluster/dask_config/reference_confs/distributed-schema.yaml +1009 -0
- cars/orchestrator/cluster/dask_config/reference_confs/distributed.yaml +273 -0
- cars/orchestrator/cluster/dask_config/reference_confs/jobqueue.yaml +212 -0
- cars/orchestrator/cluster/dask_jobqueue_utils.py +204 -0
- cars/orchestrator/cluster/local_dask_cluster.py +116 -0
- cars/orchestrator/cluster/log_wrapper.py +1075 -0
- cars/orchestrator/cluster/mp_cluster/__init__.py +27 -0
- cars/orchestrator/cluster/mp_cluster/mp_factorizer.py +212 -0
- cars/orchestrator/cluster/mp_cluster/mp_objects.py +535 -0
- cars/orchestrator/cluster/mp_cluster/mp_tools.py +93 -0
- cars/orchestrator/cluster/mp_cluster/mp_wrapper.py +505 -0
- cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py +873 -0
- cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py +399 -0
- cars/orchestrator/cluster/pbs_dask_cluster.py +207 -0
- cars/orchestrator/cluster/sequential_cluster.py +139 -0
- cars/orchestrator/cluster/slurm_dask_cluster.py +234 -0
- cars/orchestrator/orchestrator.py +905 -0
- cars/orchestrator/orchestrator_constants.py +29 -0
- cars/orchestrator/registry/__init__.py +23 -0
- cars/orchestrator/registry/abstract_registry.py +143 -0
- cars/orchestrator/registry/compute_registry.py +106 -0
- cars/orchestrator/registry/id_generator.py +116 -0
- cars/orchestrator/registry/replacer_registry.py +213 -0
- cars/orchestrator/registry/saver_registry.py +363 -0
- cars/orchestrator/registry/unseen_registry.py +118 -0
- cars/orchestrator/tiles_profiler.py +279 -0
- cars/pipelines/__init__.py +26 -0
- cars/pipelines/conf_resolution/conf_final_resolution.yaml +5 -0
- cars/pipelines/conf_resolution/conf_first_resolution.yaml +2 -0
- cars/pipelines/conf_resolution/conf_intermediate_resolution.yaml +2 -0
- cars/pipelines/default/__init__.py +26 -0
- cars/pipelines/default/default_pipeline.py +786 -0
- cars/pipelines/parameters/__init__.py +0 -0
- cars/pipelines/parameters/advanced_parameters.py +417 -0
- cars/pipelines/parameters/advanced_parameters_constants.py +69 -0
- cars/pipelines/parameters/application_parameters.py +71 -0
- cars/pipelines/parameters/depth_map_inputs.py +0 -0
- cars/pipelines/parameters/dsm_inputs.py +918 -0
- cars/pipelines/parameters/dsm_inputs_constants.py +25 -0
- cars/pipelines/parameters/output_constants.py +52 -0
- cars/pipelines/parameters/output_parameters.py +454 -0
- cars/pipelines/parameters/sensor_inputs.py +842 -0
- cars/pipelines/parameters/sensor_inputs_constants.py +49 -0
- cars/pipelines/parameters/sensor_loaders/__init__.py +29 -0
- cars/pipelines/parameters/sensor_loaders/basic_classif_loader.py +86 -0
- cars/pipelines/parameters/sensor_loaders/basic_image_loader.py +98 -0
- cars/pipelines/parameters/sensor_loaders/pivot_classif_loader.py +90 -0
- cars/pipelines/parameters/sensor_loaders/pivot_image_loader.py +105 -0
- cars/pipelines/parameters/sensor_loaders/sensor_loader.py +93 -0
- cars/pipelines/parameters/sensor_loaders/sensor_loader_template.py +71 -0
- cars/pipelines/parameters/sensor_loaders/slurp_classif_loader.py +86 -0
- cars/pipelines/pipeline.py +119 -0
- cars/pipelines/pipeline_constants.py +31 -0
- cars/pipelines/pipeline_template.py +139 -0
- cars/pipelines/unit/__init__.py +26 -0
- cars/pipelines/unit/unit_pipeline.py +2850 -0
- cars/starter.py +167 -0
- cars-1.0.0rc1.dist-info/METADATA +292 -0
- cars-1.0.0rc1.dist-info/RECORD +202 -0
- cars-1.0.0rc1.dist-info/WHEEL +5 -0
- cars-1.0.0rc1.dist-info/entry_points.txt +8 -0
- cars.libs/libgcc_s-1257a076.so.1 +0 -0
- cars.libs/libstdc++-0530927c.so.6.0.32 +0 -0
|
@@ -0,0 +1,1009 @@
|
|
|
1
|
+
properties:
|
|
2
|
+
distributed:
|
|
3
|
+
type: object
|
|
4
|
+
properties:
|
|
5
|
+
|
|
6
|
+
version:
|
|
7
|
+
type: integer
|
|
8
|
+
|
|
9
|
+
scheduler:
|
|
10
|
+
type: object
|
|
11
|
+
properties:
|
|
12
|
+
|
|
13
|
+
allowed-failures:
|
|
14
|
+
type: integer
|
|
15
|
+
minimum: 0
|
|
16
|
+
description: |
|
|
17
|
+
The number of retries before a task is considered bad
|
|
18
|
+
|
|
19
|
+
When a worker dies when a task is running that task is rerun elsewhere.
|
|
20
|
+
If many workers die while running this same task then we call the task bad, and raise a KilledWorker exception.
|
|
21
|
+
This is the number of workers that are allowed to die before this task is marked as bad.
|
|
22
|
+
|
|
23
|
+
bandwidth:
|
|
24
|
+
type:
|
|
25
|
+
- integer
|
|
26
|
+
- string
|
|
27
|
+
description: |
|
|
28
|
+
The expected bandwidth between any pair of workers
|
|
29
|
+
|
|
30
|
+
This is used when making scheduling decisions.
|
|
31
|
+
The scheduler will use this value as a baseline, but also learn it over time.
|
|
32
|
+
|
|
33
|
+
blocked-handlers:
|
|
34
|
+
type: array
|
|
35
|
+
description: |
|
|
36
|
+
A list of handlers to exclude
|
|
37
|
+
|
|
38
|
+
The scheduler operates by receiving messages from various workers and clients
|
|
39
|
+
and then performing operations based on those messages.
|
|
40
|
+
Each message has an operation like "close-worker" or "task-finished".
|
|
41
|
+
In some high security situations administrators may choose to block certain handlers
|
|
42
|
+
from running. Those handlers can be listed here.
|
|
43
|
+
|
|
44
|
+
For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
|
|
45
|
+
|
|
46
|
+
default-data-size:
|
|
47
|
+
type:
|
|
48
|
+
- string
|
|
49
|
+
- integer
|
|
50
|
+
description: |
|
|
51
|
+
The default size of a piece of data if we don't know anything about it.
|
|
52
|
+
|
|
53
|
+
This is used by the scheduler in some scheduling decisions
|
|
54
|
+
|
|
55
|
+
events-cleanup-delay:
|
|
56
|
+
type: string
|
|
57
|
+
description: |
|
|
58
|
+
The amount of time to wait until workers or clients are removed from the event log
|
|
59
|
+
after they have been removed from the scheduler
|
|
60
|
+
|
|
61
|
+
idle-timeout:
|
|
62
|
+
type:
|
|
63
|
+
- string
|
|
64
|
+
- "null"
|
|
65
|
+
description: |
|
|
66
|
+
Shut down the scheduler after this duration if no activity has occured
|
|
67
|
+
|
|
68
|
+
This can be helpful to reduce costs and stop zombie processes from roaming the earth.
|
|
69
|
+
|
|
70
|
+
transition-log-length:
|
|
71
|
+
type: integer
|
|
72
|
+
minimum: 0
|
|
73
|
+
description: |
|
|
74
|
+
How long should we keep the transition log
|
|
75
|
+
|
|
76
|
+
Every time a task transitions states (like "waiting", "processing", "memory", "released")
|
|
77
|
+
we record that transition in a log.
|
|
78
|
+
|
|
79
|
+
To make sure that we don't run out of memory
|
|
80
|
+
we will clear out old entries after a certain length.
|
|
81
|
+
This is that length.
|
|
82
|
+
|
|
83
|
+
events-log-length:
|
|
84
|
+
type: integer
|
|
85
|
+
minimum: 0
|
|
86
|
+
description: |
|
|
87
|
+
How long should we keep the events log
|
|
88
|
+
|
|
89
|
+
All events (e.g. worker heartbeat) are stored in the events log.
|
|
90
|
+
|
|
91
|
+
To make sure that we don't run out of memory
|
|
92
|
+
we will clear out old entries after a certain length.
|
|
93
|
+
This is that length.
|
|
94
|
+
|
|
95
|
+
work-stealing:
|
|
96
|
+
type: boolean
|
|
97
|
+
description: |
|
|
98
|
+
Whether or not to balance work between workers dynamically
|
|
99
|
+
|
|
100
|
+
Some times one worker has more work than we expected.
|
|
101
|
+
The scheduler will move these tasks around as necessary by default.
|
|
102
|
+
Set this to false to disable this behavior
|
|
103
|
+
|
|
104
|
+
work-stealing-interval:
|
|
105
|
+
type: string
|
|
106
|
+
description: |
|
|
107
|
+
How frequently to balance worker loads
|
|
108
|
+
|
|
109
|
+
worker-ttl:
|
|
110
|
+
type:
|
|
111
|
+
- string
|
|
112
|
+
- "null"
|
|
113
|
+
description: |
|
|
114
|
+
Time to live for workers.
|
|
115
|
+
|
|
116
|
+
If we don't receive a heartbeat faster than this then we assume that the worker has died.
|
|
117
|
+
|
|
118
|
+
pickle:
|
|
119
|
+
type: boolean
|
|
120
|
+
description: |
|
|
121
|
+
Is the scheduler allowed to deserialize arbitrary bytestrings?
|
|
122
|
+
|
|
123
|
+
The scheduler almost never deserializes user data.
|
|
124
|
+
However there are some cases where the user can submit functions to run directly on the scheduler.
|
|
125
|
+
This can be convenient for debugging, but also introduces some security risk.
|
|
126
|
+
By setting this to false we ensure that the user is unable to run arbitrary code on the scheduler.
|
|
127
|
+
|
|
128
|
+
preload:
|
|
129
|
+
type: array
|
|
130
|
+
description: |
|
|
131
|
+
Run custom modules during the lifetime of the scheduler
|
|
132
|
+
|
|
133
|
+
You can run custom modules when the scheduler starts up and closes down.
|
|
134
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
135
|
+
|
|
136
|
+
preload-argv:
|
|
137
|
+
type: array
|
|
138
|
+
description: |
|
|
139
|
+
Arguments to pass into the preload scripts described above
|
|
140
|
+
|
|
141
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
142
|
+
|
|
143
|
+
unknown-task-duration:
|
|
144
|
+
type: string
|
|
145
|
+
description: |
|
|
146
|
+
Default duration for all tasks with unknown durations
|
|
147
|
+
|
|
148
|
+
Over time the scheduler learns a duration for tasks.
|
|
149
|
+
However when it sees a new type of task for the first time it has to make a guess
|
|
150
|
+
as to how long it will take. This value is that guess.
|
|
151
|
+
|
|
152
|
+
default-task-durations:
|
|
153
|
+
type: object
|
|
154
|
+
description: |
|
|
155
|
+
How long we expect function names to run
|
|
156
|
+
|
|
157
|
+
Over time the scheduler will learn these values, but these give it a good starting point.
|
|
158
|
+
|
|
159
|
+
validate:
|
|
160
|
+
type: boolean
|
|
161
|
+
description: |
|
|
162
|
+
Whether or not to run consistency checks during execution.
|
|
163
|
+
This is typically only used for debugging.
|
|
164
|
+
|
|
165
|
+
dashboard:
|
|
166
|
+
type: object
|
|
167
|
+
description: |
|
|
168
|
+
Configuration options for Dask's real-time dashboard
|
|
169
|
+
|
|
170
|
+
properties:
|
|
171
|
+
status:
|
|
172
|
+
type: object
|
|
173
|
+
description: The main status page of the dashboard
|
|
174
|
+
properties:
|
|
175
|
+
task-stream-length:
|
|
176
|
+
type: integer
|
|
177
|
+
minimum: 0
|
|
178
|
+
description: |
|
|
179
|
+
The maximum number of tasks to include in the task stream plot
|
|
180
|
+
tasks:
|
|
181
|
+
type: object
|
|
182
|
+
description: |
|
|
183
|
+
The page which includes the full task stream history
|
|
184
|
+
properties:
|
|
185
|
+
task-stream-length:
|
|
186
|
+
type: integer
|
|
187
|
+
minimum: 0
|
|
188
|
+
description: |
|
|
189
|
+
The maximum number of tasks to include in the task stream plot
|
|
190
|
+
tls:
|
|
191
|
+
type: object
|
|
192
|
+
description: |
|
|
193
|
+
Settings around securing the dashboard
|
|
194
|
+
properties:
|
|
195
|
+
ca-file:
|
|
196
|
+
type:
|
|
197
|
+
- string
|
|
198
|
+
- "null"
|
|
199
|
+
key:
|
|
200
|
+
type:
|
|
201
|
+
- string
|
|
202
|
+
- "null"
|
|
203
|
+
cert:
|
|
204
|
+
type:
|
|
205
|
+
- string
|
|
206
|
+
- "null"
|
|
207
|
+
bokeh-application:
|
|
208
|
+
type: object
|
|
209
|
+
description: |
|
|
210
|
+
Keywords to pass to the BokehTornado application
|
|
211
|
+
locks:
|
|
212
|
+
type: object
|
|
213
|
+
description: |
|
|
214
|
+
Settings for Dask's distributed Lock object
|
|
215
|
+
|
|
216
|
+
See https://docs.dask.org/en/latest/futures.html#locks for more information
|
|
217
|
+
properties:
|
|
218
|
+
lease-validation-interval:
|
|
219
|
+
type: string
|
|
220
|
+
description: |
|
|
221
|
+
The interval in which the scheduler validates staleness of all acquired leases. Must always be smaller than the lease-timeout itself.
|
|
222
|
+
lease-timeout:
|
|
223
|
+
type: string
|
|
224
|
+
description: |
|
|
225
|
+
Maximum interval to wait for a Client refresh before a lease is invalidated and released.
|
|
226
|
+
|
|
227
|
+
http:
|
|
228
|
+
type: object
|
|
229
|
+
decription: Settings for Dask's embedded HTTP Server
|
|
230
|
+
properties:
|
|
231
|
+
routes:
|
|
232
|
+
type: array
|
|
233
|
+
description: |
|
|
234
|
+
A list of modules like "prometheus" and "health" that can be included or excluded as desired
|
|
235
|
+
|
|
236
|
+
These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
|
|
237
|
+
This is also a list that can be extended with user defined modules.
|
|
238
|
+
|
|
239
|
+
allowed-imports:
|
|
240
|
+
type: array
|
|
241
|
+
description: |
|
|
242
|
+
A list of trusted root modules the schedular is allowed to import (incl. submodules). For security reasons, the
|
|
243
|
+
scheduler does not import arbitrary Python modules.
|
|
244
|
+
|
|
245
|
+
active-memory-manager:
|
|
246
|
+
type: object
|
|
247
|
+
required: [start, interval, policies]
|
|
248
|
+
additionalProperties: false
|
|
249
|
+
properties:
|
|
250
|
+
start:
|
|
251
|
+
type: boolean
|
|
252
|
+
description: set to true to auto-start the AMM on Scheduler init
|
|
253
|
+
interval:
|
|
254
|
+
type: string
|
|
255
|
+
description:
|
|
256
|
+
Time expression, e.g. "2s". Run the AMM cycle every <interval>.
|
|
257
|
+
policies:
|
|
258
|
+
type: array
|
|
259
|
+
items:
|
|
260
|
+
type: object
|
|
261
|
+
required: [class]
|
|
262
|
+
properties:
|
|
263
|
+
class:
|
|
264
|
+
type: string
|
|
265
|
+
description: fully qualified name of an ActiveMemoryManagerPolicy
|
|
266
|
+
subclass
|
|
267
|
+
additionalProperties:
|
|
268
|
+
description: keyword arguments to the policy constructor, if any
|
|
269
|
+
|
|
270
|
+
worker:
|
|
271
|
+
type: object
|
|
272
|
+
description: |
|
|
273
|
+
Configuration settings for Dask Workers
|
|
274
|
+
properties:
|
|
275
|
+
blocked-handlers:
|
|
276
|
+
type: array
|
|
277
|
+
description: |
|
|
278
|
+
A list of handlers to exclude
|
|
279
|
+
|
|
280
|
+
The scheduler operates by receiving messages from various workers and clients
|
|
281
|
+
and then performing operations based on those messages.
|
|
282
|
+
Each message has an operation like "close-worker" or "task-finished".
|
|
283
|
+
In some high security situations administrators may choose to block certain handlers
|
|
284
|
+
from running. Those handlers can be listed here.
|
|
285
|
+
|
|
286
|
+
For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
|
|
287
|
+
|
|
288
|
+
multiprocessing-method:
|
|
289
|
+
type: string
|
|
290
|
+
description: |
|
|
291
|
+
How we create new workers, one of "spawn", "forkserver", or "fork"
|
|
292
|
+
|
|
293
|
+
This is passed to the ``multiprocessing.get_context`` function.
|
|
294
|
+
use-file-locking:
|
|
295
|
+
type: boolean
|
|
296
|
+
description: |
|
|
297
|
+
Whether or not to use lock files when creating workers
|
|
298
|
+
|
|
299
|
+
Workers create a local directory in which to place temporary files.
|
|
300
|
+
When many workers are created on the same process at once
|
|
301
|
+
these workers can conflict with each other by trying to create this directory all at the same time.
|
|
302
|
+
|
|
303
|
+
To avoid this, Dask usually used a file-based lock.
|
|
304
|
+
However, on some systems file-based locks don't work.
|
|
305
|
+
This is particularly common on HPC NFS systems, where users may want to set this to false.
|
|
306
|
+
connections:
|
|
307
|
+
type: object
|
|
308
|
+
description: |
|
|
309
|
+
The number of concurrent connections to allow to other workers
|
|
310
|
+
properties:
|
|
311
|
+
incoming:
|
|
312
|
+
type: integer
|
|
313
|
+
minimum: 0
|
|
314
|
+
outgoing:
|
|
315
|
+
type: integer
|
|
316
|
+
minimum: 0
|
|
317
|
+
|
|
318
|
+
preload:
|
|
319
|
+
type: array
|
|
320
|
+
description: |
|
|
321
|
+
Run custom modules during the lifetime of the worker
|
|
322
|
+
|
|
323
|
+
You can run custom modules when the worker starts up and closes down.
|
|
324
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
325
|
+
|
|
326
|
+
preload-argv:
|
|
327
|
+
type: array
|
|
328
|
+
description: |
|
|
329
|
+
Arguments to pass into the preload scripts described above
|
|
330
|
+
|
|
331
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
332
|
+
|
|
333
|
+
daemon:
|
|
334
|
+
type: boolean
|
|
335
|
+
description: |
|
|
336
|
+
Whether or not to run our process as a daemon process
|
|
337
|
+
|
|
338
|
+
validate:
|
|
339
|
+
type: boolean
|
|
340
|
+
description: |
|
|
341
|
+
Whether or not to run consistency checks during execution.
|
|
342
|
+
This is typically only used for debugging.
|
|
343
|
+
|
|
344
|
+
resources:
|
|
345
|
+
type: object
|
|
346
|
+
description: |
|
|
347
|
+
A dictionary specifying resources for workers.
|
|
348
|
+
|
|
349
|
+
See https://distributed.dask.org/en/latest/resources.html for more information.
|
|
350
|
+
properties: {}
|
|
351
|
+
|
|
352
|
+
lifetime:
|
|
353
|
+
type: object
|
|
354
|
+
description: |
|
|
355
|
+
The worker may choose to gracefully close itself down after some pre-determined time.
|
|
356
|
+
|
|
357
|
+
This is particularly useful if you know that your worker job has a time limit on it.
|
|
358
|
+
This is particularly common in HPC job schedulers.
|
|
359
|
+
|
|
360
|
+
For example if your worker has a walltime of one hour,
|
|
361
|
+
then you may want to set the lifetime.duration to "55 minutes"
|
|
362
|
+
properties:
|
|
363
|
+
duration:
|
|
364
|
+
type:
|
|
365
|
+
- string
|
|
366
|
+
- "null"
|
|
367
|
+
description: |
|
|
368
|
+
The time after creation to close the worker, like "1 hour"
|
|
369
|
+
stagger:
|
|
370
|
+
type: string
|
|
371
|
+
description: |
|
|
372
|
+
Random amount by which to stagger lifetimes
|
|
373
|
+
|
|
374
|
+
If you create many workers at the same time,
|
|
375
|
+
you may want to avoid having them kill themselves all at the same time.
|
|
376
|
+
To avoid this you might want to set a stagger time,
|
|
377
|
+
so that they close themselves with some random variation, like "5 minutes"
|
|
378
|
+
|
|
379
|
+
That way some workers can die, new ones can be brought up,
|
|
380
|
+
and data can be transferred over smoothly.
|
|
381
|
+
restart:
|
|
382
|
+
type: boolean
|
|
383
|
+
description: |
|
|
384
|
+
Do we try to resurrect the worker after the lifetime deadline?
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
profile:
|
|
388
|
+
type: object
|
|
389
|
+
description: |
|
|
390
|
+
The workers periodically poll every worker thread to see what they are working on.
|
|
391
|
+
This data gets collected into statistical profiling information,
|
|
392
|
+
which is then periodically bundled together and sent along to the scheduler.
|
|
393
|
+
properties:
|
|
394
|
+
interval:
|
|
395
|
+
type: string
|
|
396
|
+
description: |
|
|
397
|
+
The time between polling the worker threads, typically short like 10ms
|
|
398
|
+
cycle:
|
|
399
|
+
type: string
|
|
400
|
+
description: |
|
|
401
|
+
The time between bundling together this data and sending it to the scheduler
|
|
402
|
+
|
|
403
|
+
This controls the granularity at which people can query the profile information
|
|
404
|
+
on the time axis.
|
|
405
|
+
low-level:
|
|
406
|
+
type: boolean
|
|
407
|
+
description: |
|
|
408
|
+
Whether or not to use the libunwind and stacktrace libraries
|
|
409
|
+
to gather profiling information at the lower level (beneath Python)
|
|
410
|
+
|
|
411
|
+
To get this to work you will need to install the experimental stacktrace library at
|
|
412
|
+
|
|
413
|
+
conda install -c numba stacktrace
|
|
414
|
+
|
|
415
|
+
See https://github.com/numba/stacktrace
|
|
416
|
+
|
|
417
|
+
memory:
|
|
418
|
+
type: object
|
|
419
|
+
description: >-
|
|
420
|
+
Settings for memory management
|
|
421
|
+
properties:
|
|
422
|
+
recent-to-old-time:
|
|
423
|
+
type: string
|
|
424
|
+
description: >-
|
|
425
|
+
When there is an increase in process memory (as observed by the
|
|
426
|
+
operating system) that is not accounted for by the dask keys stored on
|
|
427
|
+
the worker, ignore it for this long before considering it in
|
|
428
|
+
non-time-sensitive heuristics. This should be set to be longer than
|
|
429
|
+
the duration of most dask tasks.
|
|
430
|
+
rebalance:
|
|
431
|
+
type: object
|
|
432
|
+
description: >-
|
|
433
|
+
Settings for memory rebalance operations
|
|
434
|
+
properties:
|
|
435
|
+
measure:
|
|
436
|
+
enum:
|
|
437
|
+
- process
|
|
438
|
+
- optimistic
|
|
439
|
+
- managed
|
|
440
|
+
- managed_in_memory
|
|
441
|
+
description: >-
|
|
442
|
+
Which of the properties of distributed.scheduler.MemoryState
|
|
443
|
+
should be used for measuring worker memory usage
|
|
444
|
+
sender-min:
|
|
445
|
+
type: number
|
|
446
|
+
minimum: 0
|
|
447
|
+
maximum: 1
|
|
448
|
+
description: >-
|
|
449
|
+
Fraction of worker process memory at which we start potentially
|
|
450
|
+
transferring data to other workers.
|
|
451
|
+
recipient-max:
|
|
452
|
+
type: number
|
|
453
|
+
minimum: 0
|
|
454
|
+
maximum: 1
|
|
455
|
+
description: >-
|
|
456
|
+
Fraction of worker process memory at which we stop potentially
|
|
457
|
+
receiving data from other workers. Ignored when max_memory is not
|
|
458
|
+
set.
|
|
459
|
+
sender-recipient-gap:
|
|
460
|
+
type: number
|
|
461
|
+
minimum: 0
|
|
462
|
+
maximum: 1
|
|
463
|
+
description: >-
|
|
464
|
+
Fraction of worker process memory, around the cluster mean, where
|
|
465
|
+
a worker is neither a sender nor a recipient of data during a
|
|
466
|
+
rebalance operation. E.g. if the mean cluster occupation is 50%,
|
|
467
|
+
sender-recipient-gap=0.1 means that only nodes above 55% will
|
|
468
|
+
donate data and only nodes below 45% will receive them. This helps
|
|
469
|
+
avoid data from bouncing around the cluster repeatedly.
|
|
470
|
+
|
|
471
|
+
target:
|
|
472
|
+
oneOf:
|
|
473
|
+
- {type: number, minimum: 0, maximum: 1}
|
|
474
|
+
- {enum: [false]}
|
|
475
|
+
description: >-
|
|
476
|
+
When the process memory (as observed by the operating system) gets
|
|
477
|
+
above this amount we start spilling the dask keys holding the largest
|
|
478
|
+
chunks of data to disk
|
|
479
|
+
|
|
480
|
+
spill:
|
|
481
|
+
oneOf:
|
|
482
|
+
- {type: number, minimum: 0, maximum: 1}
|
|
483
|
+
- {enum: [false]}
|
|
484
|
+
description: >-
|
|
485
|
+
When the process memory (as observed by the operating system) gets
|
|
486
|
+
above this amount we spill all data to disk.
|
|
487
|
+
|
|
488
|
+
pause:
|
|
489
|
+
oneOf:
|
|
490
|
+
- {type: number, minimum: 0, maximum: 1}
|
|
491
|
+
- {enum: [false]}
|
|
492
|
+
description: >-
|
|
493
|
+
When the process memory (as observed by the operating system) gets
|
|
494
|
+
above this amount we no longer start new tasks on this worker.
|
|
495
|
+
|
|
496
|
+
terminate:
|
|
497
|
+
oneOf:
|
|
498
|
+
- {type: number, minimum: 0, maximum: 1}
|
|
499
|
+
- {enum: [false]}
|
|
500
|
+
description: >-
|
|
501
|
+
When the process memory reaches this level the nanny process will kill
|
|
502
|
+
the worker (if a nanny is present)
|
|
503
|
+
|
|
504
|
+
http:
|
|
505
|
+
type: object
|
|
506
|
+
decription: Settings for Dask's embedded HTTP Server
|
|
507
|
+
properties:
|
|
508
|
+
routes:
|
|
509
|
+
type: array
|
|
510
|
+
description: |
|
|
511
|
+
A list of modules like "prometheus" and "health" that can be included or excluded as desired
|
|
512
|
+
|
|
513
|
+
These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
|
|
514
|
+
This is also a list that can be extended with user defined modules.
|
|
515
|
+
|
|
516
|
+
nanny:
|
|
517
|
+
type: object
|
|
518
|
+
description: |
|
|
519
|
+
Configuration settings for Dask Nannies
|
|
520
|
+
properties:
|
|
521
|
+
|
|
522
|
+
preload:
|
|
523
|
+
type: array
|
|
524
|
+
description: |
|
|
525
|
+
Run custom modules during the lifetime of the scheduler
|
|
526
|
+
|
|
527
|
+
You can run custom modules when the scheduler starts up and closes down.
|
|
528
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
529
|
+
|
|
530
|
+
preload-argv:
|
|
531
|
+
type: array
|
|
532
|
+
description: |
|
|
533
|
+
Arguments to pass into the preload scripts described above
|
|
534
|
+
|
|
535
|
+
See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
|
|
536
|
+
|
|
537
|
+
environ:
|
|
538
|
+
type: object
|
|
539
|
+
description: |
|
|
540
|
+
Environment variables to set on all worker processes started by nannies
|
|
541
|
+
|
|
542
|
+
client:
|
|
543
|
+
type: object
|
|
544
|
+
description: |
|
|
545
|
+
Configuration settings for Dask Clients
|
|
546
|
+
|
|
547
|
+
properties:
|
|
548
|
+
heartbeat:
|
|
549
|
+
type: string
|
|
550
|
+
description:
|
|
551
|
+
This value is the time between heartbeats
|
|
552
|
+
|
|
553
|
+
The client sends a periodic heartbeat message to the scheduler.
|
|
554
|
+
If it misses enough of these then the scheduler assumes that it has gone.
|
|
555
|
+
|
|
556
|
+
scheduler-info-interval:
|
|
557
|
+
type: string
|
|
558
|
+
description: Interval between scheduler-info updates
|
|
559
|
+
|
|
560
|
+
deploy:
|
|
561
|
+
type: object
|
|
562
|
+
description: Configuration settings for general Dask deployment
|
|
563
|
+
properties:
|
|
564
|
+
lost-worker-timeout:
|
|
565
|
+
type: string
|
|
566
|
+
description: |
|
|
567
|
+
Interval after which to hard-close a lost worker job
|
|
568
|
+
|
|
569
|
+
Otherwise we wait for a while to see if a worker will reappear
|
|
570
|
+
|
|
571
|
+
cluster-repr-interval:
|
|
572
|
+
type: string
|
|
573
|
+
description: Interval between calls to update cluster-repr for the widget
|
|
574
|
+
|
|
575
|
+
adaptive:
|
|
576
|
+
type: object
|
|
577
|
+
description: Configuration settings for Dask's adaptive scheduling
|
|
578
|
+
properties:
|
|
579
|
+
interval:
|
|
580
|
+
type: string
|
|
581
|
+
description: |
|
|
582
|
+
The duration between checking in with adaptive scheduling load
|
|
583
|
+
|
|
584
|
+
The adaptive system periodically checks scheduler load and determines
|
|
585
|
+
if it should scale the cluster up or down.
|
|
586
|
+
This is the timing between those checks.
|
|
587
|
+
|
|
588
|
+
target-duration:
|
|
589
|
+
type: string
|
|
590
|
+
description: |
|
|
591
|
+
The desired time for the entire computation to run
|
|
592
|
+
|
|
593
|
+
The adaptive system will try to start up enough workers to run
|
|
594
|
+
the computation in about this time.
|
|
595
|
+
|
|
596
|
+
minimum:
|
|
597
|
+
type: integer
|
|
598
|
+
minimum: 0
|
|
599
|
+
description: |
|
|
600
|
+
The minimum number of workers to keep around
|
|
601
|
+
|
|
602
|
+
maximum:
|
|
603
|
+
type: number
|
|
604
|
+
minimum: 0
|
|
605
|
+
description: |
|
|
606
|
+
The maximum number of workers to keep around
|
|
607
|
+
|
|
608
|
+
wait-count:
|
|
609
|
+
type: integer
|
|
610
|
+
minimum: 1
|
|
611
|
+
description: |
|
|
612
|
+
The number of times a worker should be suggested for removal before removing it
|
|
613
|
+
|
|
614
|
+
This helps to smooth out the number of deployed workers
|
|
615
|
+
|
|
616
|
+
comm:
|
|
617
|
+
type: object
|
|
618
|
+
description: Configuration settings for Dask communications
|
|
619
|
+
properties:
|
|
620
|
+
|
|
621
|
+
retry:
|
|
622
|
+
type: object
|
|
623
|
+
description: |
|
|
624
|
+
Some operations (such as gathering data) are subject to re-tries with the below parameters
|
|
625
|
+
properties:
|
|
626
|
+
|
|
627
|
+
count:
|
|
628
|
+
type: integer
|
|
629
|
+
minimum: 0
|
|
630
|
+
description: |
|
|
631
|
+
The number of times to retry a connection
|
|
632
|
+
|
|
633
|
+
delay:
|
|
634
|
+
type: object
|
|
635
|
+
properties:
|
|
636
|
+
min:
|
|
637
|
+
type: string
|
|
638
|
+
description: The first non-zero delay between retry attempts
|
|
639
|
+
max:
|
|
640
|
+
type: string
|
|
641
|
+
description: The maximum delay between retries
|
|
642
|
+
|
|
643
|
+
compression:
|
|
644
|
+
type: string
|
|
645
|
+
description: |
|
|
646
|
+
The compression algorithm to use
|
|
647
|
+
|
|
648
|
+
This could be one of lz4, snappy, zstd, or blosc
|
|
649
|
+
|
|
650
|
+
offload:
|
|
651
|
+
type:
|
|
652
|
+
- boolean
|
|
653
|
+
- string
|
|
654
|
+
description: |
|
|
655
|
+
The size of message after which we choose to offload serialization to another thread
|
|
656
|
+
|
|
657
|
+
In some cases, you may also choose to disable this altogether with the value false
|
|
658
|
+
This is useful if you want to include serialization in profiling data,
|
|
659
|
+
or if you have data types that are particularly sensitive to deserialization
|
|
660
|
+
|
|
661
|
+
shard:
|
|
662
|
+
type: string
|
|
663
|
+
description: |
|
|
664
|
+
The maximum size of a frame to send through a comm
|
|
665
|
+
|
|
666
|
+
Some network infrastructure doesn't like sending through very large messages.
|
|
667
|
+
Dask comms will cut up these large messages into many small ones.
|
|
668
|
+
This attribute determines the maximum size of such a shard.
|
|
669
|
+
|
|
670
|
+
socket-backlog:
|
|
671
|
+
type: integer
|
|
672
|
+
description: |
|
|
673
|
+
When shuffling data between workers, there can
|
|
674
|
+
really be O(cluster size) connection requests
|
|
675
|
+
on a single worker socket, make sure the backlog
|
|
676
|
+
is large enough not to lose any.
|
|
677
|
+
|
|
678
|
+
zstd:
|
|
679
|
+
type: object
|
|
680
|
+
description: Options for the Z Standard compression scheme
|
|
681
|
+
properties:
|
|
682
|
+
level:
|
|
683
|
+
type: integer
|
|
684
|
+
minimum: 1
|
|
685
|
+
maximum: 22
|
|
686
|
+
description: Compression level, between 1 and 22.
|
|
687
|
+
threads:
|
|
688
|
+
type: integer
|
|
689
|
+
minimum: -1
|
|
690
|
+
description: |
|
|
691
|
+
Number of threads to use.
|
|
692
|
+
|
|
693
|
+
0 for single-threaded, -1 to infer from cpu count.
|
|
694
|
+
|
|
695
|
+
timeouts:
|
|
696
|
+
type: object
|
|
697
|
+
properties:
|
|
698
|
+
connect:
|
|
699
|
+
type: string
|
|
700
|
+
tcp:
|
|
701
|
+
type: string
|
|
702
|
+
|
|
703
|
+
require-encryption:
|
|
704
|
+
type:
|
|
705
|
+
- boolean
|
|
706
|
+
- "null"
|
|
707
|
+
description: |
|
|
708
|
+
Whether to require encryption on non-local comms
|
|
709
|
+
|
|
710
|
+
default-scheme:
|
|
711
|
+
type: string
|
|
712
|
+
description: The default protocol to use, like tcp or tls
|
|
713
|
+
|
|
714
|
+
recent-messages-log-length:
|
|
715
|
+
type: integer
|
|
716
|
+
minimum: 0
|
|
717
|
+
description: number of messages to keep for debugging
|
|
718
|
+
|
|
719
|
+
tls:
|
|
720
|
+
type: object
|
|
721
|
+
properties:
|
|
722
|
+
ciphers:
|
|
723
|
+
type:
|
|
724
|
+
- string
|
|
725
|
+
- "null"
|
|
726
|
+
description: Allowed ciphers, specified as an OpenSSL cipher string.
|
|
727
|
+
|
|
728
|
+
min-version:
|
|
729
|
+
enum: [null, 1.2, 1.3]
|
|
730
|
+
description: The minimum TLS version to support. Defaults to TLS 1.2.
|
|
731
|
+
|
|
732
|
+
max-version:
|
|
733
|
+
enum: [null, 1.2, 1.3]
|
|
734
|
+
description: |
|
|
735
|
+
The maximum TLS version to support. Defaults to the maximum
|
|
736
|
+
version supported by the platform.
|
|
737
|
+
|
|
738
|
+
ca-file:
|
|
739
|
+
type:
|
|
740
|
+
- string
|
|
741
|
+
- "null"
|
|
742
|
+
description: Path to a CA file, in pem format
|
|
743
|
+
|
|
744
|
+
scheduler:
|
|
745
|
+
type: object
|
|
746
|
+
description: TLS information for the scheduler
|
|
747
|
+
properties:
|
|
748
|
+
cert:
|
|
749
|
+
type:
|
|
750
|
+
- string
|
|
751
|
+
- "null"
|
|
752
|
+
description: Path to certificate file
|
|
753
|
+
key:
|
|
754
|
+
type:
|
|
755
|
+
- string
|
|
756
|
+
- "null"
|
|
757
|
+
description: |
|
|
758
|
+
Path to key file.
|
|
759
|
+
|
|
760
|
+
Alternatively, the key can be appended to the cert file
|
|
761
|
+
above, and this field left blank
|
|
762
|
+
|
|
763
|
+
worker:
|
|
764
|
+
type: object
|
|
765
|
+
description: TLS information for the worker
|
|
766
|
+
properties:
|
|
767
|
+
cert:
|
|
768
|
+
type:
|
|
769
|
+
- string
|
|
770
|
+
- "null"
|
|
771
|
+
description: Path to certificate file
|
|
772
|
+
key:
|
|
773
|
+
type:
|
|
774
|
+
- string
|
|
775
|
+
- "null"
|
|
776
|
+
description: |
|
|
777
|
+
Path to key file.
|
|
778
|
+
|
|
779
|
+
Alternatively, the key can be appended to the cert file
|
|
780
|
+
above, and this field left blank
|
|
781
|
+
|
|
782
|
+
client:
|
|
783
|
+
type: object
|
|
784
|
+
description: TLS information for the client
|
|
785
|
+
properties:
|
|
786
|
+
cert:
|
|
787
|
+
type:
|
|
788
|
+
- string
|
|
789
|
+
- "null"
|
|
790
|
+
description: Path to certificate file
|
|
791
|
+
key:
|
|
792
|
+
type:
|
|
793
|
+
- string
|
|
794
|
+
- "null"
|
|
795
|
+
description: |
|
|
796
|
+
Path to key file.
|
|
797
|
+
|
|
798
|
+
Alternatively, the key can be appended to the cert file
|
|
799
|
+
above, and this field left blank
|
|
800
|
+
|
|
801
|
+
ucx:
|
|
802
|
+
type: object
|
|
803
|
+
description: |
|
|
804
|
+
UCX provides access to other transport methods including NVLink and InfiniBand.
|
|
805
|
+
properties:
|
|
806
|
+
cuda-copy:
|
|
807
|
+
type: [boolean, 'null']
|
|
808
|
+
description: |
|
|
809
|
+
Set environment variables to enable CUDA support over UCX. This may be used even if
|
|
810
|
+
InfiniBand and NVLink are not supported or disabled, then transferring data over TCP.
|
|
811
|
+
tcp:
|
|
812
|
+
type: [boolean, 'null']
|
|
813
|
+
description: |
|
|
814
|
+
Set environment variables to enable TCP over UCX, even if InfiniBand and NVLink
|
|
815
|
+
are not supported or disabled.
|
|
816
|
+
nvlink:
|
|
817
|
+
type: [boolean, 'null']
|
|
818
|
+
description: |
|
|
819
|
+
Set environment variables to enable UCX over NVLink, implies ``distributed.comm.ucx.tcp=True``.
|
|
820
|
+
infiniband:
|
|
821
|
+
type: [boolean, 'null']
|
|
822
|
+
description: |
|
|
823
|
+
Set environment variables to enable UCX over InfiniBand, implies ``distributed.comm.ucx.tcp=True``.
|
|
824
|
+
rdmacm:
|
|
825
|
+
type: [boolean, 'null']
|
|
826
|
+
description: |
|
|
827
|
+
Set environment variables to enable UCX RDMA connection manager support,
|
|
828
|
+
requires ``distributed.comm.ucx.infiniband=True``.
|
|
829
|
+
net-devices:
|
|
830
|
+
type: [string, 'null']
|
|
831
|
+
description: |
|
|
832
|
+
Interface(s) used by workers for UCX communication. Can be a string (like
|
|
833
|
+
``"eth0"`` for NVLink or ``"mlx5_0:1"``/``"ib0"`` for InfiniBand), ``"auto"``
|
|
834
|
+
(requires ``distributed.comm.ucx.infiniband=True``) to pick the optimal interface per-worker based on
|
|
835
|
+
the system's topology, or ``None`` to stay with the default value of ``"all"`` (use
|
|
836
|
+
all available interfaces). Setting to ``"auto"`` requires UCX-Py to be installed
|
|
837
|
+
and compiled with hwloc support. Unexpected errors can occur when using
|
|
838
|
+
``"auto"`` if any interfaces are disconnected or improperly configured.
|
|
839
|
+
reuse-endpoints:
|
|
840
|
+
type: [boolean, 'null']
|
|
841
|
+
description: |
|
|
842
|
+
Enable UCX-Py reuse endpoints mechanism if ``True`` or if it's not specified and
|
|
843
|
+
UCX < 1.11 is installed, otherwise disable reuse endpoints. This was primarily
|
|
844
|
+
introduced to resolve an issue with CUDA IPC that has been fixed in UCX 1.10, but
|
|
845
|
+
can cause establishing endpoints to be very slow, this is particularly noticeable in
|
|
846
|
+
clusters of more than a few dozen workers.
|
|
847
|
+
create-cuda-context:
|
|
848
|
+
type: [boolean, 'null']
|
|
849
|
+
description: |
|
|
850
|
+
Creates a CUDA context before UCX is initialized. This is necessary to enable UCX to
|
|
851
|
+
properly identify connectivity of GPUs with specialized networking hardware, such as
|
|
852
|
+
InfiniBand. This permits UCX to choose transports automatically, without specifying
|
|
853
|
+
additional variables for each transport, while ensuring optimal connectivity. When
|
|
854
|
+
``True``, a CUDA context will be created on the first device listed in
|
|
855
|
+
``CUDA_VISIBLE_DEVICES``.
|
|
856
|
+
|
|
857
|
+
tcp:
|
|
858
|
+
type: object
|
|
859
|
+
properties:
|
|
860
|
+
backend:
|
|
861
|
+
type: string
|
|
862
|
+
description: |
|
|
863
|
+
The TCP backend implementation to use. Must be either `tornado` or `asyncio`.
|
|
864
|
+
|
|
865
|
+
websockets:
|
|
866
|
+
type: object
|
|
867
|
+
properties:
|
|
868
|
+
shard:
|
|
869
|
+
type:
|
|
870
|
+
- string
|
|
871
|
+
description: |
|
|
872
|
+
The maximum size of a websocket frame to send through a comm.
|
|
873
|
+
|
|
874
|
+
This is somewhat duplicative of distributed.comm.shard, but websockets
|
|
875
|
+
often have much smaller maximum message sizes than othe protocols, so
|
|
876
|
+
this attribute is used to set a smaller default shard size and to
|
|
877
|
+
allow separate control of websocket message sharding.
|
|
878
|
+
|
|
879
|
+
diagnostics:
|
|
880
|
+
type: object
|
|
881
|
+
properties:
|
|
882
|
+
nvml:
|
|
883
|
+
type: boolean
|
|
884
|
+
description: |
|
|
885
|
+
If ``True``, enables GPU diagnostics with NVML. Generally leaving it enabled is
|
|
886
|
+
not a problem and will be automatically disabled if no GPUs are found in the
|
|
887
|
+
system, but in certain cases it may be desirable to completely disable NVML
|
|
888
|
+
diagnostics.
|
|
889
|
+
computations:
|
|
890
|
+
type: object
|
|
891
|
+
properties:
|
|
892
|
+
max-history:
|
|
893
|
+
type: integer
|
|
894
|
+
minimum: 0
|
|
895
|
+
description: |
|
|
896
|
+
The maximum number of Computations to remember.
|
|
897
|
+
ignore-modules:
|
|
898
|
+
type: array
|
|
899
|
+
description: |
|
|
900
|
+
A list of modules which are ignored when trying to collect the
|
|
901
|
+
code context when submitting a computation. Accepts regular
|
|
902
|
+
expressions.
|
|
903
|
+
|
|
904
|
+
dashboard:
|
|
905
|
+
type: object
|
|
906
|
+
properties:
|
|
907
|
+
link:
|
|
908
|
+
type: string
|
|
909
|
+
description: |
|
|
910
|
+
The form for the dashboard links
|
|
911
|
+
|
|
912
|
+
This is used wherever we print out the link for the dashboard
|
|
913
|
+
It is filled in with relevant information like the schema, host, and port number
|
|
914
|
+
graph-max-items:
|
|
915
|
+
type: integer
|
|
916
|
+
minimum: 0
|
|
917
|
+
description: maximum number of tasks to try to plot in "graph" view
|
|
918
|
+
|
|
919
|
+
export-tool:
|
|
920
|
+
type: boolean
|
|
921
|
+
|
|
922
|
+
prometheus:
|
|
923
|
+
type: object
|
|
924
|
+
properties:
|
|
925
|
+
namespace:
|
|
926
|
+
type: string
|
|
927
|
+
description: Namespace prefix to use for all prometheus metrics.
|
|
928
|
+
|
|
929
|
+
admin:
|
|
930
|
+
type: object
|
|
931
|
+
description: |
|
|
932
|
+
Options for logs, event loops, and so on
|
|
933
|
+
properties:
|
|
934
|
+
tick:
|
|
935
|
+
type: object
|
|
936
|
+
description: |
|
|
937
|
+
Time between event loop health checks
|
|
938
|
+
|
|
939
|
+
We set up a periodic callback to run on the event loop and check in fairly frequently.
|
|
940
|
+
(by default, this is every 20 milliseconds)
|
|
941
|
+
|
|
942
|
+
If this periodic callback sees that the last time it checked in was several seconds ago
|
|
943
|
+
(by default, this is 3 seconds)
|
|
944
|
+
then it logs a warning saying that something has been stopping the event loop from smooth operation.
|
|
945
|
+
This is typically caused by GIL holding operations,
|
|
946
|
+
but could also be several other things.
|
|
947
|
+
|
|
948
|
+
properties:
|
|
949
|
+
interval:
|
|
950
|
+
type: string
|
|
951
|
+
description: The time between ticks, default 20ms
|
|
952
|
+
limit :
|
|
953
|
+
type: string
|
|
954
|
+
description: The time allowed before triggering a warning
|
|
955
|
+
|
|
956
|
+
max-error-length:
|
|
957
|
+
type: integer
|
|
958
|
+
minimum: 0
|
|
959
|
+
description: |
|
|
960
|
+
Maximum length of traceback as text
|
|
961
|
+
|
|
962
|
+
Some Python tracebacks can be very very long
|
|
963
|
+
(particularly in stack overflow errors)
|
|
964
|
+
|
|
965
|
+
If the traceback is larger than this size (in bytes) then we truncate it.
|
|
966
|
+
|
|
967
|
+
log-length:
|
|
968
|
+
type: integer
|
|
969
|
+
minimum: 0
|
|
970
|
+
description: |
|
|
971
|
+
Default length of logs to keep in memory
|
|
972
|
+
|
|
973
|
+
The scheduler and workers keep the last 10000 or so log entries in memory.
|
|
974
|
+
|
|
975
|
+
log-format:
|
|
976
|
+
type: string
|
|
977
|
+
description: |
|
|
978
|
+
The log format to emit.
|
|
979
|
+
|
|
980
|
+
See https://docs.python.org/3/library/logging.html#logrecord-attributes
|
|
981
|
+
event-loop:
|
|
982
|
+
type: string
|
|
983
|
+
description: |
|
|
984
|
+
The event loop to use,
|
|
985
|
+
|
|
986
|
+
Must be one of tornado, asyncio, or uvloop
|
|
987
|
+
|
|
988
|
+
pdb-on-err:
|
|
989
|
+
type: boolean
|
|
990
|
+
description: Enter Python Debugger on scheduling error
|
|
991
|
+
|
|
992
|
+
system-monitor:
|
|
993
|
+
type: object
|
|
994
|
+
description: |
|
|
995
|
+
Options for the periodic system monitor
|
|
996
|
+
properties:
|
|
997
|
+
interval:
|
|
998
|
+
type: string
|
|
999
|
+
description: Polling time to query cpu/memory statistics default 500ms
|
|
1000
|
+
|
|
1001
|
+
rmm:
|
|
1002
|
+
type: object
|
|
1003
|
+
description: |
|
|
1004
|
+
Configuration options for the RAPIDS Memory Manager.
|
|
1005
|
+
properties:
|
|
1006
|
+
pool-size:
|
|
1007
|
+
type: [integer, 'null']
|
|
1008
|
+
description: |
|
|
1009
|
+
The size of the memory pool in bytes.
|