cars 1.0.0rc1__cp313-cp313-musllinux_1_2_i686.whl

cars/orchestrator/cluster/dask_config/reference_confs/distributed.yaml

@@ -0,0 +1,273 @@
+distributed:
+  version: 2
+  # logging:
+  #   distributed: info
+  #   distributed.client: warning
+  #   bokeh: critical
+  #   # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
+  #   tornado: critical
+  #   tornado.application: error
+
+  scheduler:
+    allowed-failures: 3     # number of retries before a task is considered bad
+    bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
+    blocked-handlers: []
+    default-data-size: 1kiB
+    # Number of seconds to wait until workers or clients are removed from the events log
+    # after they have been removed from the scheduler
+    events-cleanup-delay: 1h
+    idle-timeout: null      # Shut down after this duration, like "1h" or "30 minutes"
+    transition-log-length: 100000
+    events-log-length: 100000
+    work-stealing: True     # workers should steal tasks from each other
+    work-stealing-interval: 100ms  # Callback time for work stealing
+    worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
+    pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
+    preload: []             # Run custom modules with Scheduler
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
+    unknown-task-duration: 500ms  # Default duration for all tasks with unknown durations ("15m", "2h")
+    default-task-durations:  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
+      rechunk-split: 1us
+      split-shuffle: 1us
+    validate: False         # Check scheduler state at every step for debugging
+    dashboard:
+      status:
+        task-stream-length: 1000
+      tasks:
+        task-stream-length: 100000
+      tls:
+        ca-file: null
+        key: null
+        cert: null
+      bokeh-application:  # keywords to pass to BokehTornado application
+        allow_websocket_origin: ["*"]
+        keep_alive_milliseconds: 500
+        check_unused_sessions_milliseconds: 500
+    locks:
+      lease-validation-interval: 10s  # The interval in which the scheduler validates staleness of all acquired leases. Must always be smaller than the lease-timeout itself.
+      lease-timeout: 30s  # Maximum interval to wait for a Client refresh before a lease is invalidated and released.
+
+    http:
+      routes:
+        - distributed.http.scheduler.prometheus
+        - distributed.http.scheduler.info
+        - distributed.http.scheduler.json
+        - distributed.http.health
+        - distributed.http.proxy
+        - distributed.http.statics
+
+    allowed-imports:
+      - dask
+      - distributed
+
+    active-memory-manager:
+      # Set to true to auto-start the Active Memory Manager on Scheduler start; if false
+      # you'll have to either manually start it with client.amm.start() or run it once
+      # with client.amm.run_once().
+      start: false
+      # Once started, run the AMM cycle every <interval>
+      interval: 2s
+      policies:
+        # Policies that should be executed at every cycle. Any additional keys in each
+        # object are passed as keyword arguments to the policy constructor.
+        - class: distributed.active_memory_manager.ReduceReplicas
+
+  worker:
+    blocked-handlers: []
+    multiprocessing-method: spawn
+    use-file-locking: True
+    connections:            # Maximum concurrent connections for data
+      outgoing: 50          # This helps to control network saturation
+      incoming: 10
+    preload: []             # Run custom modules with Worker
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
+    daemon: True
+    validate: False         # Check worker state at every step for debugging
+    resources: {}           # Key: value pairs specifying worker resources.
+    lifetime:
+      duration: null        # Time after which to gracefully shutdown the worker
+      stagger: 0 seconds    # Random amount by which to stagger lifetimes
+      restart: False        # Do we ressurrect the worker after the lifetime deadline?
+
+    profile:
+      interval: 10ms        # Time between statistical profiling queries
+      cycle: 1000ms         # Time between starting new profile
+      low-level: False      # Whether or not to include low-level functions
+                            # Requires https://github.com/numba/stacktrace
+
+    memory:
+      # When there is an increase in process memory (as observed by the operating
+      # system) that is not accounted for by the dask keys stored on the worker, ignore
+      # it for this long before considering it in non-critical memory measures.
+      # This should be set to be longer than the duration of most dask tasks.
+      recent-to-old-time: 30s
+
+      rebalance:
+        # Memory measure to rebalance upon. Possible choices are:
+        # process
+        #     Total process memory, as measured by the OS.
+        # optimistic
+        #     Managed by dask (instantaneous) + unmanaged (without any increases
+        #     happened in the last <distributed.worker.memory.recent-to-old-time>).
+        #     Recommended for use on CPython with large (2MiB+) numpy-based data chunks.
+        # managed_in_memory
+        #     Only consider the data allocated by dask in RAM. Recommended if RAM is not
+        #     released in a timely fashion back to the OS after the Python objects are
+        #     dereferenced, but remains available for reuse by PyMalloc.
+        #
+        #     If this is your problem on Linux, you should alternatively consider
+        #     setting the MALLOC_TRIM_THRESHOLD_ environment variable (note the final
+        #     underscore) to a low value; refer to the mallopt man page and to the
+        #     comments about M_TRIM_THRESHOLD on
+        #     https://sourceware.org/git/?p=glibc.git;a=blob;f=malloc/malloc.c
+        # managed
+        #     Only consider data allocated by dask, including that spilled to disk.
+        #     Recommended if disk occupation of the spill file is an issue.
+        measure: optimistic
+        # Fraction of worker process memory at which we start potentially sending
+        # data to other workers. Ignored when max_memory is not set.
+        sender-min: 0.30
+        # Fraction of worker process memory at which we stop potentially accepting
+        # data from other workers. Ignored when max_memory is not set.
+        recipient-max: 0.60
+        # Fraction of worker process memory, around the cluster mean, where a worker is
+        # neither a sender nor a recipient of data during a rebalance operation. E.g. if
+        # the mean cluster occupation is 50%, sender-recipient-gap=0.10 means that only
+        # nodes above 55% will donate data and only nodes below 45% will receive them.
+        # This helps avoid data from bouncing around the cluster repeatedly.
+        # Ignored when max_memory is not set.
+        sender-recipient-gap: 0.10
+
+      # Fractions of worker process memory at which we take action to avoid memory
+      # blowup. Set any of the values to False to turn off the behavior entirely.
+      target: 0.60  # target fraction to stay below
+      spill: 0.70  # fraction at which we spill to disk
+      pause: 0.80  # fraction at which we pause worker threads
+      terminate: 0.95  # fraction at which we terminate the worker
+
+    http:
+      routes:
+        - distributed.http.worker.prometheus
+        - distributed.http.health
+        - distributed.http.statics
+
+  nanny:
+    preload: []             # Run custom modules with Nanny
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
+    environ:
+      MALLOC_TRIM_THRESHOLD_: 0
+      OMP_NUM_THREADS: 1
+      MKL_NUM_THREADS: 1
+
+  client:
+    heartbeat: 5s  # Interval between client heartbeats
+    scheduler-info-interval: 2s  # Interval between scheduler-info updates
+
+  deploy:
+    lost-worker-timeout: 15s  # Interval after which to hard-close a lost worker job
+    cluster-repr-interval: 500ms  # Interval between calls to update cluster-repr for the widget
+
+  adaptive:
+    interval: 1s         # Interval between scaling evaluations
+    target-duration: 5s  # Time an entire graph calculation is desired to take ("1m", "30m")
+    minimum: 0           # Minimum number of workers
+    maximum: .inf        # Maximum number of workers
+    wait-count: 3        # Number of times a worker should be suggested for removal before removing it
+
+  comm:
+    retry:  # some operations (such as gathering data) are subject to re-tries with the below parameters
+      count: 0  # the maximum retry attempts. 0 disables re-trying.
+      delay:
+         min: 1s  # the first non-zero delay between re-tries
+         max: 20s  # the maximum delay between re-tries
+    compression: auto
+    shard: 64MiB
+    offload: 10MiB # Size after which we choose to offload serialization to another thread
+    default-scheme: tcp
+    socket-backlog: 2048
+    recent-messages-log-length: 0  # number of messages to keep for debugging
+    ucx:
+      cuda-copy: null  # enable cuda-copy
+      tcp: null  # enable tcp
+      nvlink: null  # enable cuda_ipc
+      infiniband: null  # enable Infiniband
+      rdmacm: null  # enable RDMACM
+      net-devices: null  # define what interface to use for UCX comm
+      reuse-endpoints: null  # enable endpoint reuse
+      create-cuda-context: null  # create CUDA context before UCX initialization
+
+    zstd:
+      level: 3      # Compression level, between 1 and 22.
+      threads: 0    # Threads to use. 0 for single-threaded, -1 to infer from cpu count.
+
+    timeouts:
+      connect: 30s          # time before connecting fails
+      tcp: 30s              # time before calling an unresponsive connection dead
+
+    require-encryption: null # Whether to require encryption on non-local comms
+
+    tls:
+      ciphers: null     # Allowed ciphers, specified as an OpenSSL cipher string.
+      min-version: 1.2  # The minimum TLS version supported.
+      max-version: null # The maximum TLS version supported.
+      ca-file: null     # Path to a CA file, in pem format, optional
+      scheduler:
+        cert: null      # Path to certificate file for scheduler.
+        key: null       # Path to key file for scheduler. Alternatively, the key
+                        # can be appended to the cert file above, and this field
+                        # left blank.
+      worker:
+        key: null
+        cert: null
+      client:
+        key: null
+        cert: null
+
+    tcp:
+      backend: tornado  # The backend to use for TCP, one of {tornado, asyncio}
+
+    websockets:
+      shard: 8MiB
+
+  diagnostics:
+    nvml: True
+    computations:
+      max-history: 100
+      ignore-modules:
+        - distributed
+        - dask
+        - xarray
+        - cudf
+        - cuml
+        - prefect
+        - xgboost
+
+  ###################
+  # Bokeh dashboard #
+  ###################
+
+  dashboard:
+    link: "{scheme}://{host}:{port}/status"
+    export-tool: False
+    graph-max-items: 5000  # maximum number of tasks to try to plot in graph view
+    prometheus:
+      namespace: "dask"
+
+  ##################
+  # Administrative #
+  ##################
+
+  admin:
+    tick:
+      interval: 20ms  # time between event loop health checks
+      limit: 3s       # time allowed before triggering a warning
+
+    max-error-length: 10000 # Maximum size traceback after error to return
+    log-length: 10000  # default length of logs to keep in memory
+    log-format: '%(name)s - %(levelname)s - %(message)s'
+    pdb-on-err: False       # enter debug mode on scheduling error
+    system-monitor:
+      interval: 500ms
+    event-loop: tornado
+  rmm:
+    pool-size: null

cars/orchestrator/cluster/dask_config/reference_confs/jobqueue.yaml

@@ -0,0 +1,212 @@
+jobqueue:
+  oar:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # OAR resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30:00'
+    env-extra: []
+    resource-spec: null
+    job-extra: []
+    log-directory: null
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  pbs:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # PBS resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30:00'
+    env-extra: []
+    resource-spec: null
+    job-extra: []
+    log-directory: null
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  sge:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # SGE resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30:00'
+    env-extra: []
+    job-extra: []
+    log-directory: null
+    resource-spec: null
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  slurm:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # SLURM resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30:00'
+    env-extra: []
+    job-cpu: null
+    job-mem: null
+    job-extra: []
+    log-directory: null
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  moab:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # PBS resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30:00'
+    env-extra: []
+    resource-spec: null
+    job-extra: []
+    log-directory: null
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  lsf:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # LSF resource manager options
+    shebang: "#!/usr/bin/env bash"
+    queue: null
+    project: null
+    walltime: '00:30'
+    env-extra: []
+    ncpus: null
+    mem: null
+    job-extra: []
+    log-directory: null
+    lsf-units: null
+    use-stdin: True             # (bool) How jobs are launched, i.e. 'bsub jobscript.sh' or 'bsub < jobscript.sh'
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  htcondor:
+    name: dask-worker
+
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    # HTCondor Resource Manager options
+    disk: null                  # Total amount of disk per job
+    env-extra: []
+    job-extra: {}               # Extra submit attributes
+    submit-command-extra: []    # Extra condor_submit arguments
+    cancel-command-extra: []    # Extra condor_rm arguments
+    log-directory: null
+    shebang: "#!/usr/bin/env condor_submit"
+    
+    # Scheduler options
+    scheduler-options: {}
+
+  local:
+    name: dask-worker
+    # Dask worker options
+    cores: null                 # Total number of cores per job
+    memory: null                # Total amount of memory per job
+    processes: null                # Number of Python processes per job
+
+    interface: null             # Network interface to use like eth0 or ib0
+    death-timeout: 60           # Number of seconds to wait if a worker can not find a scheduler
+    local-directory: null       # Location of fast local storage like /scratch or $TMPDIR
+    shared-temp-directory: null       # Shared directory currently used to dump temporary security objects for workers
+    extra: []
+
+    env-extra: []
+    job-extra: []
+    log-directory: null
+    
+    # Scheduler options
+    scheduler-options: {}

cars/orchestrator/cluster/dask_jobqueue_utils.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains functions for dask jobqueue cluster (PBS, SLURM)
+"""
+
+import logging
+import math
+import os
+import sys
+import warnings
+from datetime import timedelta
+
+
+def init_cluster_variables(  # pylint: disable=too-many-positional-arguments
+    nb_workers,
+    walltime,
+    out_dir,
+    activate_dashboard,
+    python,
+    core_memory,
+    cluster_name,
+):
+    """
+    Initialize global cluster variables
+    :param nb_workers: number of workers
+    :param walltime: workers walltime
+    :param out_dir: ouput result directory
+    :param activate_dashboard: option to activate dashboard mode
+    :param python: target python used by workers (retrun system python if None)
+    :param core_memory: cluster node memory (Mo)
+    :param cluster_name: PBS or SLURM
+    :return: all cluster parameters (python,
+    nb_workers_per_job,
+    memory,
+    nb_cpus,
+    stagger,
+    lifetime_with_margin,
+    scheduler_options,
+    envs,
+    log_directory,
+    local_directory)
+    """
+
+    if python is None:
+        python = sys.executable
+
+    # Configure worker distribution.
+    # Workers are mostly running single-threaded, GIL-holding functions, so we
+    # dedicate a single thread for each worker to maximize CPU utilization.
+    nb_threads_per_worker = 1
+
+    # Network latency is not the bottleneck, so we dedicate a single worker for
+    # each job in order to minimize the requested resources, which reduces our
+    # scheduling delay.
+    nb_workers_per_job = 1
+
+    # Total number of CPUs is multi-threading factor times size of batch
+    # (number of workers per job)
+    nb_cpus = nb_threads_per_worker * nb_workers_per_job
+    nb_jobs = int(math.ceil(nb_workers / nb_workers_per_job))
+    # Cluster nodes have core_memory Mo per core
+    memory = nb_cpus * core_memory
+
+    # Configure worker lifetime for adaptative scaling.
+    # See https://jobqueue.dask.org/en/latest/advanced-tips-and-tricks.html
+    hours, minutes, seconds = map(int, walltime.split(":"))
+    lifetime = timedelta(seconds=3600 * hours + 60 * minutes + seconds)
+    # Use hardcoded stagger of 3 minutes. The actual lifetime will be selected
+    # uniformly at random between lifetime +/- stagger.
+    stagger = timedelta(minutes=3)
+    # Add some margin to not get killed by scheduler during worker shutdown.
+    shutdown_margin = timedelta(minutes=2)
+    min_walltime = stagger + shutdown_margin
+    lifetime_with_margin = lifetime - min_walltime
+    if lifetime_with_margin.total_seconds() < 0:
+        min_walltime_minutes = min_walltime.total_seconds() / 60
+        logging.warning(
+            "Could not add worker lifetime margin because specified walltime "
+            "is too short. Workers might get killed by {} before they can "
+            "cleanly exit, which might break adaptative scaling. Please "
+            "specify a lifetime greater than {} minutes.".format(
+                cluster_name, min_walltime_minutes
+            )
+        )
+        lifetime_with_margin = lifetime
+
+    logging.info(
+        "Starting Dask {0} cluster with {1} workers "
+        "({2} workers with {3} cores each per {0} job)".format(
+            cluster_name,
+            nb_workers,
+            nb_workers_per_job,
+            nb_threads_per_worker,
+        )
+    )
+
+    logging.info(
+        "Submitting {} {} jobs "
+        "with configuration cpu={}, mem={}, walltime={}".format(
+            nb_jobs, cluster_name, nb_cpus, memory, walltime
+        )
+    )
+
+    if activate_dashboard:
+        scheduler_options = None
+    else:
+        scheduler_options = {"dashboard": None, "dashboard_address": None}
+
+    names = [
+        "PATH",
+        "PYTHONPATH",
+        "CARS_STATIC_CONFIGURATION",
+        "LD_LIBRARY_PATH",
+        "OTB_APPLICATION_PATH",
+        "OTB_MAX_RAM_HINT",
+        "OMP_NUM_THREADS",
+        "OPJ_NUM_THREADS",
+        "GDAL_NUM_THREADS",
+        "VIRTUAL_ENV",
+        "ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS",
+        "GDAL_CACHEMAX",
+        "DASK_CONFIG",
+        "NUMBA_THREADING_LAYER",
+    ]
+    names = [name for name in names if os.environ.get(name)]
+    envs = ["export {}={}".format(name, os.environ[name]) for name in names]
+    log_directory = os.path.join(os.path.abspath(out_dir), "dask_log")
+    local_directory = "$TMPDIR"
+    return (
+        python,
+        nb_workers_per_job,
+        memory,
+        nb_cpus,
+        stagger,
+        lifetime_with_margin,
+        scheduler_options,
+        envs,
+        log_directory,
+        local_directory,
+    )
+
+
+def get_dashboard_link(cluster):
+    """
+    This function returns the dashboard address.
+
+    :param cluster: Dask cluster
+    :type cluster: dask_jobqueue.PBSCluster
+    :return: Link to the dashboard
+    :rtype: string
+    """
+    template = "http://{host}:{port}/status"
+    host = cluster.scheduler.address.split("://")[1].split(":")[0]
+    port = cluster.scheduler.services["dashboard"].port
+    return template.format(host=host, port=port)
+
+
+def stop_cluster(cluster, client):
+    """
+    This function stops a dask cluster.
+
+    :param cluster: Dask cluster
+    :type cluster: dask_jobqueue.PBSCluster
+    :param client: Dask client
+    :type client: dask.distributed.Client
+    """
+    client.close()
+    # Bug distributed on close cluster : Fail with AssertionError still running
+    try:
+        with warnings.catch_warnings():
+            # Ignore internal dask_jobqueue warnings to be corrected in:
+            # https://github.com/dask/dask-jobqueue/pull/506
+            warnings.filterwarnings(
+                "ignore",
+                category=FutureWarning,
+                message=".*ignoring was deprecated in version 2021.06.1.*",
+            )
+            cluster.close()
+    except AssertionError as assert_error:
+        logging.warning(
+            "Dask cluster failed " "to stop properly: {}".format(assert_error)
+        )
+        # not raising to not fail tests
+
+    logging.info("Dask cluster correctly stopped")