cars 1.0.0rc1__cp313-cp313-musllinux_1_2_i686.whl

cars/orchestrator/cluster/abstract_dask_cluster.py

@@ -0,0 +1,375 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains abstract function for abstract dask Cluster
+"""
+
+# Standard imports
+import logging
+import os
+import time
+
+# Third party imports
+from abc import abstractmethod
+
+import dask
+import numpy as np
+import psutil
+import xarray as xr
+import yaml
+from dask.config import global_config as global_dask_config
+from dask.config import set as dask_config_set
+from dask.delayed import Delayed
+from dask.distributed import as_completed
+from dask.sizeof import sizeof as dask_sizeof
+from distributed.diagnostics.plugin import WorkerPlugin
+from distributed.utils import CancelledError
+
+from cars.core import cars_logging
+
+# CARS imports
+from cars.orchestrator.cluster import abstract_cluster
+
+
+class AbstractDaskCluster(
+    abstract_cluster.AbstractCluster
+):  # pylint: disable=R0902
+    """
+    AbstractDaskCluster
+    """
+
+    def __init__(self, conf_cluster, out_dir, log_dir, launch_worker=True):
+        """
+        Init function of AbstractDaskCluster
+
+        :param conf_cluster: configuration for cluster
+
+        """
+
+        print("cluster log_dir", log_dir)
+
+        # call parent init
+        super().__init__(
+            conf_cluster, out_dir, log_dir, launch_worker=launch_worker
+        )
+        # retrieve parameters
+        self.nb_workers = self.checked_conf_cluster["nb_workers"]
+        self.task_timeout = self.checked_conf_cluster["task_timeout"]
+        self.walltime = self.checked_conf_cluster["walltime"]
+        self.use_memory_logger = self.checked_conf_cluster["use_memory_logger"]
+        self.config_name = self.checked_conf_cluster["config_name"]
+        self.profiling = self.checked_conf_cluster["profiling"]
+        self.launch_worker = launch_worker
+
+        self.activate_dashboard = self.checked_conf_cluster[
+            "activate_dashboard"
+        ]
+        self.python = self.checked_conf_cluster["python"]
+        if self.checked_conf_cluster["mode"] == "slurm_dask":
+            self.account = self.checked_conf_cluster["account"]
+            self.qos = self.checked_conf_cluster["qos"]
+
+        if self.launch_worker:
+            # Set DASK CARS specific config
+            # TODO: update with adequate configuration through tests
+            set_config()
+
+            # Save dask config used
+            save_config(self.out_dir, "dask_config_" + self.config_name)
+
+            # Create cluster
+            self.cluster, self.client = self.start_dask_cluster()
+
+            # Add plugin to monitor memory of workers
+            if self.use_memory_logger:
+                plugin = ComputeDSMMemoryLogger(self.out_dir)
+                self.client.register_worker_plugin(plugin)
+
+    @abstractmethod
+    def check_conf(self, conf):
+        """
+        Check configuration
+
+        :param conf: configuration to check
+        :type conf: dict
+
+        :return: overloaded configuration
+        :rtype: dict
+
+        """
+
+    @abstractmethod
+    def start_dask_cluster(self):
+        """
+        Start dask cluster
+        """
+
+    def create_task_wrapped(self, func, nout=1):
+        """
+        Create task
+
+        :param func: function
+        :param nout: number of outputs
+        """
+        return dask.delayed(
+            cars_logging.wrap_logger(func, self.worker_log_dir, self.log_level),
+            nout=nout,
+        )
+
+    def get_delayed_type(self):
+        """
+        Get delayed type
+        """
+        return Delayed
+
+    def start_tasks(self, task_list):
+        """
+        Start all tasks
+
+        :param task_list: task list
+        """
+
+        return self.client.compute(task_list)
+
+    def scatter(self, data, broadcast=True):
+        """
+        Distribute data through workers
+
+        :param data: task data
+        """
+        return self.client.scatter(data, broadcast=broadcast)
+
+    def future_iterator(self, future_list, timeout=None):
+        """
+        Start all tasks
+
+        :param future_list: future_list list
+        """
+
+        return DaskFutureIterator(future_list, timeout=timeout)
+
+
+class DaskFutureIterator:
+    """
+    iterator on dask futures, similar to as_completed
+    Only returns the actual results, delete the future after usage
+    """
+
+    def __init__(self, future_list, timeout=None):  # pylint: disable=W0613
+        # TODO: python 3.9: add timeout=timeout as parameter
+        self.dask_a_c = as_completed(future_list, with_results=True)
+        self.prev = None
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            fut, res = self.dask_a_c.__next__()
+        except StopIteration as exception:
+            if self.prev is not None:
+                self.prev.cancel()
+                self.prev = None
+            raise exception
+        except dask.distributed.TimeoutError as exception:
+            raise TimeoutError("No tasks available") from exception
+        # release previous future
+        if self.prev is not None:
+            self.prev.cancel()
+            # store current future
+            self.prev = fut
+
+        if isinstance(res, CancelledError):
+            raise RuntimeError("CancelError from worker {}".format(res))
+        return res
+
+
+def set_config():
+    """
+    Set particular DASK config such as:
+    - scheduler
+    """
+    # TODO: export API to prepare.run and compute_dsm.run() to set scheduler
+    # example mode debug: dask_config_set(scheduler='single-threaded')
+    # example mode multithread: dask_config_set(scheduler='threads')
+    # Here set Multiprocess mode instead multithread because of GIL blocking
+    dask_config_set(scheduler="processes")
+
+
+def save_config(output_dir: str, file_name: str):
+    """
+    Save DASK global config
+
+    :param output_dir: output directory path
+    :param file_name: output file name
+
+    """
+    logging.info(
+        "Save DASK global merged config for debug "
+        "(1: $DASK_DIR if exists, 2: ~/.config/dask/, ... ) "
+    )
+    # write global merged DASK config in YAML
+    write_yaml_config(global_dask_config, output_dir, file_name)
+
+
+def write_yaml_config(yaml_config: dict, output_dir: str, file_name: str):
+    """
+    Writes a YAML config to disk.
+    TODO: put in global file if needed elsewhere than DASK conf save.
+
+    :param yaml_config: YAML config to write
+    :param output_dir: output directory path
+    :param file_name: output file name
+    """
+    # file path where to store the dask config
+    yaml_config_path = os.path.join(output_dir, file_name + ".yaml")
+    with open(yaml_config_path, "w", encoding="utf-8") as yaml_config_file:
+        yaml.dump(yaml_config, yaml_config_file)
+
+
+@dask_sizeof.register_lazy("xarray")
+def register_xarray():
+    """
+    Add hook to dask so it correctly estimates memory used by xarray
+    """
+
+    @dask_sizeof.register(xr.DataArray)
+    # pylint: disable=unused-variable
+    def sizeof_xarray_dataarray(xarr):
+        """
+        Inner function for total size of xarray_dataarray
+        """
+        total_size = dask_sizeof(xarr.values)
+        for __, carray in xarr.coords.items():
+            total_size += dask_sizeof(carray.values)
+        total_size += dask_sizeof(xarr.attrs)
+        return total_size
+
+    @dask_sizeof.register(xr.Dataset)
+    # pylint: disable=unused-variable
+    def sizeof_xarray_dataset(xdat):
+        """
+        Inner function for total size of xarray_dataset
+        """
+        total_size = 0
+        for __, varray in xdat.data_vars.items():
+            total_size += dask_sizeof(varray.values)
+        for __, carray in xdat.coords.items():
+            total_size += dask_sizeof(carray)
+        total_size += dask_sizeof(xdat.attrs)
+        return total_size
+
+
+class ComputeDSMMemoryLogger(WorkerPlugin):
+    """A subclass of WorkerPlugin dedicated to monitoring workers memory
+
+    This plugin enables two things:
+
+    - Additional dask log traces (for each worker internal state change):
+
+        - amount of tasks
+        - associated memory
+    - A numpy data file with memory metrics and timing
+    """
+
+    def __init__(self, outdir):
+        """
+        Constructor
+        :param outdir: output directory
+        :type outdir: string
+        """
+        self.outdir = outdir
+
+    def setup(self, worker):
+        """
+        Associate plugin with a worker
+        :param worker: The worker to associate the plugin with
+        """
+        # Pylint Exception : Inherited attributes outside __init__
+        # pylint: disable=attribute-defined-outside-init
+        self.worker = worker
+        self.name = worker.name
+        # Measure plugin registration time
+        self.start_time = time.time()
+        # Data will hold the memory traces as numpy array
+        self.data = [[0, 0, 0, 0]]
+
+    def transition(self, key, start, finish, **kwargs):
+        """
+        Callback when worker changes internal state
+        """
+        # TODO Pylint Exception : Inherited attributes outside __init__
+        # pylint: disable=attribute-defined-outside-init
+
+        # Define cumulants
+        total_in_memory = 0
+        total_nbytes = 0
+
+        # Measure elapsed time for the state change
+        elapsed_time = time.time() - self.start_time
+
+        # Walk the worker known memory
+        for task_key in self.worker.state.tasks.keys():
+            task_size = self.worker.state.tasks[task_key].get_nbytes()
+
+            total_in_memory += task_size
+            total_nbytes += 1
+
+        # Use psutil to capture python process memory as well
+        process = psutil.Process(os.getpid())
+        process_memory = process.memory_info().rss
+
+        # Update data records
+        self.data = np.concatenate(
+            (
+                self.data,
+                np.array(
+                    [
+                        [
+                            elapsed_time,
+                            total_in_memory,
+                            total_nbytes,
+                            process_memory,
+                        ]
+                    ]
+                ),
+            )
+        )
+        # Convert nbytes size for logger
+        total_nbytes = float(total_nbytes) / 1000000
+        process_memory = float(process_memory) / 1000000
+
+        # Log memory state
+        logging.info(
+            "Memory report: data created = {} ({} Mb), "
+            "python process memory = {} Mb".format(
+                total_in_memory,
+                total_nbytes,
+                process_memory,
+            )
+        )
+
+        # Save data records in npy file
+        # TODO: Save only every x seconds ?
+        file = os.path.join(
+            self.outdir, "dask_log", "memory_" + repr(self.name) + ".npy"
+        )
+        np.save(file, self.data)

cars/orchestrator/cluster/dask_cluster_tools.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains functions cluster conf checker
+"""
+import time
+
+from json_checker import And, Checker, Or
+
+
+def create_checker_schema(conf):
+    """
+    Create checker shema that it can be overloaded
+
+    :param conf: configuration to check
+    :type conf: dict
+
+    :return: overloaded configuration
+    :rtype: dict
+
+    """
+
+    # init conf
+    if conf is not None:
+        overloaded_conf = conf.copy()
+    else:
+        conf = {}
+        overloaded_conf = {}
+
+    # Overload conf
+    overloaded_conf["mode"] = conf.get("mode", "unknowed_dask")
+    overloaded_conf["use_memory_logger"] = conf.get("use_memory_logger", False)
+    overloaded_conf["nb_workers"] = conf.get("nb_workers", 2)
+    overloaded_conf["task_timeout"] = conf.get("task_timeout", 600)
+    overloaded_conf["max_ram_per_worker"] = conf.get("max_ram_per_worker", 2000)
+    overloaded_conf["walltime"] = conf.get("walltime", "00:59:00")
+    overloaded_conf["config_name"] = conf.get("config_name", "unknown")
+    overloaded_conf["activate_dashboard"] = conf.get(
+        "activate_dashboard", False
+    )
+    overloaded_conf["python"] = conf.get("python", None)
+    overloaded_conf["profiling"] = conf.get("profiling", {})
+
+    cluster_schema = {
+        "mode": str,
+        "use_memory_logger": bool,
+        "nb_workers": And(int, lambda x: x > 0),
+        "task_timeout": And(int, lambda x: x > 0),
+        "max_ram_per_worker": And(Or(float, int), lambda x: x > 0),
+        "walltime": str,
+        "config_name": str,
+        "activate_dashboard": bool,
+        "profiling": dict,
+        "python": Or(None, str),
+    }
+
+    return overloaded_conf, cluster_schema
+
+
+def check_configuration(overloaded_conf, cluster_schema):
+    """
+    Check configuration from overload conf and cluster schema
+
+
+    :param conf: overloaded_conf to check
+    :type conf: dict
+    :param conf: cluster_schema checking rules
+    :type conf: dict
+
+    :return: overloaded configuration
+    :rtype: dict
+
+    """
+    # Check conf
+    checker = Checker(cluster_schema)
+    checker.validate(overloaded_conf)
+
+    # Check walltime format
+    walltime = overloaded_conf["walltime"]
+    try:
+        time.strptime(walltime, "%H:%M:%S")
+    except ValueError as err:
+        raise ValueError("Walltime should be formatted as HH:MM:SS") from err
+
+    return overloaded_conf

cars/orchestrator/cluster/dask_config/README.md

@@ -0,0 +1,94 @@
+# CARS Dask configuration
+
+This file contains the description of DASK configuration used with CARS
+
+This configuration is split into 3 files:
+- [dask.yaml](./dask.yaml) : basic DASK configuration
+- [distributed.yaml](./distributed.yaml) : configuration of DASK distributed : scheduler and workers
+- [jobqueue.yaml](./jobqueue.yaml) : configuration of DASK and PBS cluster interface
+
+
+##  dask.yaml configuration
+
+
+| Option  | CARS value  | Default Dask value  | Comments |
+| ------- | ----------- | ------------------- |  -------- |
+| temporary-directory | null | None = null | Default: tmpdir = dask-worker-space local |
+| dataframe:shuffle-compression| null | None = null  | Default:  no compression . TODO: test if gain |
+| array:svg:size | 120 | 120  | Default. |
+| array:slicing:split-large-chunks| null | None = null | Default.  TODO: set to true if too much warnings. Test if adds perfomance issues |
+| optimization:fuse:active| **true** | None = null | **Activated** TODO: test impact on configuration activation  |
+| optimization:fuse:ave-width| 1| 1  |  Default. **TODO : to test/adapt limit of width= num_nodes/height** |
+| optimization:fuse:max-width| null| None = null  | Default : depends on de ave-width. 1.5 + ave_width * log(ave_width + 1) |
+| optimization:fuse:max-height| .inf | inf  | Default : No max limitation. All possibilities. |
+| optimization:fuse:max-depth-new-edges| null|None = null | Default :  depends on ave-width. ave_width * 1.5 |
+| optimization:fuse:subgraphs| null | None = null | Default: Let optimizer choose. TODO: test?|
+| optimization:fuse:rename-keys| true | true  |  Default. TODO: test ?|
+
+## Configuration distributed.yaml
+
+
+| Option  | CARS value  | Default Dask value  | Comments |
+| ------- | ----------- | ------------------- | -------- |
+| version | **2** | NA| Not defined in documentation | Default in example. TODO: keep ? why specify ?  no documentation...   |
+| logging:distributed| **info** | info |  DASK distributed log level. *Adapt depending the output loglevel. Set to debug for mac level* |
+|  logging:distributed.client| warning | warning | See for Client, adapt of refined analysis |
+| logging:distributed.worker| debug | not set  | Default. For more workers infors : uncomment.|
+| logging:bokeh | **critical** | error ?  |  Log limitation on warnings / errors polluting output.  |
+| logging:tornado | **critical** | warning ?  | Duplicate in output configuration .  To test in dask.yaml. Link with  http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr ?. The idea is to delete tornado messages, but does not talk about dask conf|
+| logging:tornado.application | **error** | ? | dig for more performance analysis of tornado (cf timeout TCP ...), We need to upgrade tornato loglevel. TODO |
+| scheduler:blocked-handlers|  [] | []  | Default. |
+| scheduler:allowed-failures| **10** | 3  | Augmented for some failed cases. TODO: lower this value |
+| scheduler:bandwidth| **100000** | 100000000 | Tested values, set lower : 100mbps vs 100G ? estimated flow. enough ? TODO: analyse the use of this parameter. |
+| scheduler:transition-log-length| 100000 | 100000 | Default. Unit? TODO: Storage size in spinning logs. Seems ok if memory doent freeze  |
+| scheduler:work-stealing| True | True  | Default. load balancing of workers tasks. TODO: Could be dangerous for stability, working with images  |
+| worker:blocked-handlers|  [] | []  | Default  |
+| :worker:multiprocessing-method| **forkserver** | spawn  | Tested modification. TODO:  see the gain with forkserver. Perf vs stability ... spawn vs fork vs forkserver. Seems ok with forkserver  as intermediate solution but maybe not that slow with spawn, ans more stable, with software stacked on each multiprocessed worker.  |
+| worker:connections:outgoing et incoming| 50 and **50**  | 50 and 10 | Default for outgoing. Modified to 50 for incoming.  TODO: Precise analysis of conections, current connections. Beware not to saturate. |
+| worker:validate| **true**  | false |  Augmentation for debug Default.  TODO: see impact? |
+| worker:lifetime:restart | **True** | False | Commented. TODO: to test. What is deadline ? See lifetime utilisation. "Lifetime was intended to provide a mechanism for a periodic reset of workers, such as might be useful when dealing with libraries that might leak memory and so benefit from cycling worker processes. It is orthogonal to adaptive." mrocklin https://github.com/dask/distributed/issues/3141 . Does lifetime as to be set the same as walltime  ? |
+| worker:lifetime:duration| null  | None=null  | Default. |
+| worker:lifetime:stagger| 0  | 0  | Default. |
+| worker:profile:interval| 10 ms  | 10ms | Default. |
+| worker:profile:cycle| 1000 ms  | 1000ms  | Default. |
+| worker:memory:target| 0.60  | 0.6 | Default.  |
+| worker:memory:spill| 0.70  | 0.7  | Default.  |
+| worker:memory:pause| 0.80  | 0.8  | Default.  |
+| worker:memory:terminate| 0.95 | 0.95  | Default.  |
+| comm:retry:count| **10** | 0  |  Tested for stability improvement. TODO: Default: 0 should not change that. For now we keep it to 10. May be lower comminuration issue between workers |
+| comm:compression| auto | auto   | Default. |
+| comm:default-scheme| tcp  | tcp  | Default. |
+| comm:socket-backlog| 2048  | 2048  | Default.  TODO: not sufficient ? |
+| comm:timeouts:connects| **60s** | 10s   | Augmentation of value following done tests. TODO: issues before. Do not touch it ... |
+| comm:timeouts:tcp| **120s** | 30s  |  Augmentation of value following done tests. TODO: issues before. Do not touch it ...  |
+| comm:require-encryption| **False** | None=null  | Deactivated. No need |
+| dashboard:export-tool| False | False   | Default |
+| admin:tick:interval | 20 ms | 20 ms | Default. |
+| admin:tick:limit| **1s** | 3s |  TODO: test it : default 3s, keep 1s ? |
+| admin:log-format|  %(asctime)s :: %(name)s - %(levelname)s - %(message)s' | %(name)s - %(levelname)s - %(message)s | default ok |
+| admin:pdb-on-err| False| False | Default. TODO test : set to true for debug. |
+
+## Configuration jobqueue.yaml
+
+The main configuration is done with [dask usage in  CARS](https://github.com/CNES/cars/blob/master/cars/orchestrator/cluster/pbs_dask_cluster.py)
+
+But also in documented file [jobqueue.yaml](./jobqueue.yaml) if not overloaded in CARS.
+[Jobqueue documentation](https://jobqueue.dask.org/en/latest/configuration.html)
+
+## Official configurations : 
+
+Reference dask configuration of dask projects are stored in [reference_confs](./reference_confs/)
+
+### Dask
+
+- default: https://raw.githubusercontent.com/dask/dask/main/dask/dask.yaml
+- schema: https://raw.githubusercontent.com/dask/dask/main/dask/dask-schema.yaml
+
+### Distributed
+
+- default: https://raw.githubusercontent.com/dask/distributed/main/distributed/distributed.yaml
+- schema: https://raw.githubusercontent.com/dask/distributed/main/distributed/distributed-schema.yaml
+
+### Jobqueue
+
+- default: https://raw.githubusercontent.com/dask/dask-jobqueue/main/dask_jobqueue/jobqueue.yaml

cars/orchestrator/cluster/dask_config/dask.yaml

@@ -0,0 +1,21 @@
+temporary-directory: null  # Directory for local disk like /tmp, /scratch, or /local. null -> local dask-worker-space directory.
+
+dataframe:
+  shuffle:
+    compression: null  # compression for on disk-shuffling. Partd supports ZLib, BZ2, SNAPPY, BLOSC
+
+array:
+  svg:
+    size: 120  # pixels for jupyter notebook array rendering
+  slicing:
+    split-large-chunks: null  # How to handle large output chunks in slicing. Warns by default.
+
+optimization:
+  fuse:  # Options for Dask's task fusion optimizations
+    active: true
+    ave-width: 1  # Upper limit for width, where width = num_nodes / height
+    max-width: null  # 1.5 + ave_width * log(ave_width + 1)
+    max-height: .inf # Fuse all possibilities
+    max-depth-new-edges: null  # ave_width * 1.5
+    subgraphs: null  # true for dask.dataframe, false for everything else.  Set to null to let the default optimizer of individual dask collections decide.
+    rename-keys: true # Set to true to rename the fused keys with `default_fused_keys_renamer`. Can be costly.

cars/orchestrator/cluster/dask_config/distributed.yaml

@@ -0,0 +1,70 @@
+distributed:
+  version: 2
+  logging:  # Logging dask distributed module conf
+    distributed: warning
+    distributed.client: warning
+    distributed.worker: warning
+    distributed.scheduler: warning
+    distributed.nanny: warning
+    distributed.core: warning
+    bokeh: critical
+    # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
+    tornado: critical
+    tornado.application: error
+
+  scheduler:
+    blocked-handlers: []   # message operation list to block from scheduler to worker
+    allowed-failures: 10     # number of retries before a task is considered bad
+    bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
+    work-stealing: True     # workers should steal tasks from each other
+
+  worker:
+    blocked-handlers: []   # message operation list to block from scheduler to worker
+    multiprocessing-method: forkserver  # can be fork, spawn or forkserver multiprocessing method
+    connections:            # Maximum concurrent connections for data
+      outgoing: 50          # This helps to control network saturation
+      incoming: 50
+    validate: True         # Check worker state at every step for debugging
+    lifetime:
+      duration: null
+      stagger: 0            # Random amount of time by which to stagger lifetime. Avoid kill at the same lifetime
+      restart: False        # Do we ressurrect the worker after the lifetime deadline?
+    profile:
+      interval: 10ms        # Time between statistical profiling queries
+      cycle: 1000ms         # Time between starting new profile
+    memory:
+      target: False  # target fraction to stay below
+      spill: False  # fraction at which we spill to disk
+      pause: False  # fraction at which we pause worker threads
+      terminate: False  # fraction at which we terminate the worker
+  comm:
+    retry:  # some operations (such as gathering data) are subject to re-tries with the below parameters
+      count: 10  # the maximum retry attempts. 0 disables re-trying.
+    compression: auto # Compression for communication; Default: auto
+    default-scheme: tcp # default scheme (can be tls for secure comm)
+    socket-backlog: 2048 # default value, must be large enough for data between workers
+
+    timeouts:
+      connect: 60s          # time before connecting fails (default: 10s)
+      tcp: 120s              # time before calling an unresponsive connection dead (default: 30s)
+
+    require-encryption: False # Whether to require encryption on non-local comms
+
+  ###################
+  # Bokeh dashboard #
+  ###################
+
+  dashboard:
+    export-tool: False  # Deactivate bokeh dashboard for performance
+
+  ##################
+  # Administrative #
+  ##################
+
+  admin:
+    tick:
+      interval: 20ms  # time between event loop health checks
+      limit: 1s       # time allowed before triggering a warning
+    log-format: '%(asctime)s :: %(name)s - %(levelname)s - %(message)s'
+    pdb-on-err: False       # enter debug mode on scheduling error
+    low-level-log-length: 100000 # How long should we keep the transition log in memory (default length 100000 (bytes?))

cars/orchestrator/cluster/dask_config/jobqueue.yaml

@@ -0,0 +1,26 @@
+jobqueue:
+  pbs:
+    name: cars-dask-worker      # Name of dask worker (set to PBS JOB NAME)
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null             # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 3000         # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    extra: []                   # Additional arguments to pass to `dask-worker`
+
+    # PBS resource manager options
+    shebang: "#!/usr/bin/env bash"  # Path to desired interpreter for your batch submission script.
+    queue: null                     # PBS queue by default. Take the one specified in PBSCluster() first
+    project: null                   # PBS project name in ACCOUNT information
+    walltime: '00:59:00'            # PBS walltime by default.
+    env-extra: []                   # PBS environnements to pass to workers
+    resource-spec: null             # Resource string for PBS cpu and memory by default
+    job-extra: []                   # ?
+    log-directory: null             # Log directory by default
+
+    # Scheduler options
+    scheduler-options: {}           # Used to pass additional arguments to Dask Scheduler.