cohere-ui 4.3__py3-none-any.whl

cohere_ui/api/auto_data.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+
+# #########################################################################
+# Copyright (c) , UChicago Argonne, LLC. All rights reserved.             #
+#                                                                         #
+# See LICENSE file.                                                       #
+# #########################################################################
+
+"""
+This file contains suite of scripts related to auto_data setting.
+While preprocessing with auto option the scripts determine which scans are outliers, i.e. have
+the greatest correlation error with relation to all other scans in the set.
+"""
+
+__author__ = "Paul Frosik"
+__docformat__ = 'restructuredtext en'
+__all__ = ['get_ref_correlation_err',
+           'find_outliers_in_batch',
+           'find_outlier_scans']
+
+import os
+import importlib
+import cohere_core.utilities as ut
+import cohere_core.utilities.dvc_utils as dvut
+import shutil
+from multiprocessing import Queue, Process, Pool
+from functools import partial
+
+
+def set_lib(pkg):
+    # initialize the library to cupy if available, otherwise to numpy
+    global devlib
+    if pkg == 'cp':
+        devlib = importlib.import_module('cohere_core.lib.cplib').cplib
+    else:
+        devlib = importlib.import_module('cohere_core.lib.nplib').nplib
+    dvut.set_lib_from_pkg(pkg)
+
+
+def get_ref_correlation_err(experiment_dir, scans, scan):
+    """
+    This function finds a mean of correlation errors calculated between given scan and all other scans.
+
+    :param experiment_dir: str
+        path to cohere experiment
+    :param scans: list of int
+        list of scans included in the batch
+    :param scan: int
+        scan number the mean correlation error is calculated for
+    :return: float
+        mean of all correlation errors
+    """
+    refarr = ut.read_tif(ut.join(experiment_dir, f'scan_{str(scan)}', 'preprocessed_data', 'prep_data.tif'))
+    err = 0
+    refarr = devlib.from_numpy(refarr)
+
+    for s in scans:
+       if s != scan:
+            datafile = ut.join(experiment_dir, f'scan_{str(s)}', 'preprocessed_data', 'prep_data.tif')
+            arr = devlib.from_numpy(ut.read_tif(datafile))
+            e = dvut.correlation_err(refarr, arr)
+            err += e
+    return (err / (len(scans) - 1), scan)
+
+
+def find_outliers_in_batch(experiment_dir, scans, q, no_processes):
+    """
+    Used by auto-data. This function is called after experiment data has been read for each scan that is part of batch, i.e. scans that are being added together to bear a data file.
+    Each scan is aligned with other scans and correlation error is calculated for each pair. The errors are summed for each scan. Mertics such as average and standard deviation on the summed errors are used to find the outliers.
+    The scans with summed errors exceeding standard deviation are considered outliers that are returned in a list. The outliers scans will be excluded from the data set.
+    The outliers scans are added to the queue and will be consumed by calling process.
+
+    :param experiment_dir: str
+        path to the cohere experiment
+    :param scans: list
+        list of scans in the batch
+    :param q: Queue
+        a queue used to pass outliers scans calculated for this batch
+    :param no_processes: int
+        number processes allocated to this computing
+    :return:
+    """
+    from statistics import mean, pstdev
+
+    err_scan = []
+    outlier_scans = []
+    # if multiple processes can run concurrently use this code
+    if no_processes > 1:
+        func = partial(get_ref_correlation_err, experiment_dir, scans)
+        with Pool(processes=no_processes) as pool:
+            res = pool.map_async(func, scans)
+            pool.close()
+            pool.join()
+        for r in res.get():
+            err_scan.append(r)
+    else:
+        # otherwise run it sequentially
+        for scan in scans:
+            err_scan.append(get_ref_correlation_err(experiment_dir, scans, scan))
+
+    err = [el[0].item() for el in err_scan]
+    err_mean = mean(err)
+    stdev = pstdev(err)
+    # print('mean, std', mean, stdev)
+    for (err_value, scan) in err_scan:
+       # print(err_value, scan)
+        if err_value > (err_mean + stdev):
+            outlier_scans.append(scan)
+    q.put(outlier_scans)
+
+
+def find_outlier_scans(experiment_dir, scans_datainfo, separate_ranges):
+    """
+    This function finds batches of scans with number of scans greater than 3 and follows to find outliers in those batches.
+    Scans data are read and saved in scan directories.
+    The function finds available resources and calls concurrent processes on each batch to find outliers.
+    The outliers scans are received through queue from each process.
+    :param experiment_dir:
+         path to the cohere experiment
+    :param read_scan_func:
+        function to read a scan data
+    :return: list of int
+        list of outliers scans
+    """
+    def remove_scan_dirs():
+        # remove individual scan directories
+        for scan_dir in os.listdir(experiment_dir):
+            if scan_dir.startswith('scan'):
+                shutil.rmtree(ut.join(experiment_dir, scan_dir))
+
+    if separate_ranges:
+        auto_batches = [batch for batch in scans_datainfo if len(batch) > 3]
+        if len(auto_batches) == 0:
+            remove_scan_dirs()
+            return []
+    else:
+        auto_batches = [s_d for batch in scans_datainfo for s_d in batch]
+        if len(auto_batches) <= 3:
+            remove_scan_dirs()
+            return []
+        else:
+            # make it a single sub-list
+            auto_batches = [auto_batches]
+
+    print('finding outliers')
+
+    # find all (scan, directory) tuples in auto_batches
+    single_scans_dinfo = [s_d for batch in auto_batches for s_d in batch]
+    # process_separate_scans(read_scan_func, single_scans_dinfo, experiment_dir)
+    #
+    # this code determines which library to use and how many scans can be processed concurrently
+    try:
+        import cupy
+        pkg = 'cp'
+        data_size = ut.read_tif(ut.join(experiment_dir, f'scan_{str(single_scans_dinfo[0][0])}', 'preprocessed_data', 'prep_data.tif')).size
+        job_size = data_size * 67 / 1000000. + 84 # empirically found constants
+        # use the first GPU
+        avail_devs_dict = ut.get_avail_gpu_runs(job_size, [0])
+        avail_devs = []
+        for k,v in avail_devs_dict.items():
+            avail_devs.extend([k] * v)
+        available_processes = len(avail_devs)
+    except:
+        pkg = 'np'
+        available_processes = os.cpu_count() * 2
+    set_lib(pkg)
+
+    # the available processes will be distributed among processes for each batch, i.e. scan range
+    no_concurrent = available_processes // len(auto_batches)
+        # in case when number of batches is greater than available processes
+        # the chunking will handle all batches
+
+    # find outliers in each batch
+    q = Queue()
+
+    outliers = []
+    chunk_size = available_processes
+    while auto_batches:
+        chunk, auto_batches = auto_batches[:chunk_size], auto_batches[chunk_size:]
+
+        processes = []
+        for batch in chunk:
+            scans_in_batch = [s_d[0] for s_d in batch]
+            p = Process(target=find_outliers_in_batch, args=(experiment_dir, scans_in_batch, q, no_concurrent))
+            processes.append(p)
+            p.start()
+        i = len(processes)
+        while i > 0:
+            outliers.extend(q.get())
+            i -= 1
+
+        for p in processes:
+            p.join()
+
+    # remove individual scan directories
+    remove_scan_dirs()
+
+    outliers.sort()
+    return outliers
+
+
+# def auto_separate_scans(experiment_dir, prep_obj, no_auto_batches):
+#     # this code determines which library to use and how many scans can be processed concurrently
+#     try:
+#         import cupy
+#         lib = 'cp'
+#         no_concurrent = 1
+#     except:
+#         lib = 'np'
+#         # the available processes will be distributed among processes for each batch, i.e. scan range
+#         no_concurrent = os.cpu_count() * 2 // no_auto_batches
+#     set_lib(lib)
+#
+#     print('finding outliers')
+#     dirs = []
+#     scans = []
+#     arrs = []
+#     batches = prep_obj.get_batches()
+#     for batch in batches:
+#         dirs.extend(batch[0])
+#         scans.extend(batch[1])
+#     # for dir in dirs:
+#     #     arr = devlib.from_numpy(prep_obj.read_scan(dir))
+#     #     arrs.append(arr)
+#     arr = devlib.from_numpy(prep_obj.read_scan(dirs[0]))
+#     arrs.append((arr))
+#     arr1 = devlib.from_numpy(prep_obj.read_scan(dirs[1]))
+#     arrs.append((arr1))
+#     # # save scans
+#     # process_separate_scans(prep_obj, dirs, scans, experiment_dir)
+#     print(scans)
+#     errs = []
+#     refarr = prep_obj.read_scan(dirs[0])
+#     for dir in dirs[1:]:
+#         arr = prep_obj.read_scan(dir)
+#         errs.append(dvut.correlation_err(devlib.from_numpy(refarr), devlib.from_numpy(arr)))
+#         refarr = arr
+#     errs = [e.item() for e in errs]
+#     for i in range(0,len(errs)):
+#         print(scans[i+1], errs[i])
+#     refarr = prep_obj.read_scan(dirs[3])
+#     arr = prep_obj.read_scan(dirs[7])
+#     print(errs)
+#
+

cohere_ui/api/balancer.py

@@ -0,0 +1,254 @@
+# #########################################################################
+# Copyright (c) , UChicago Argonne, LLC. All rights reserved.             #
+#                                                                         #
+# See LICENSE file.                                                       #
+# #########################################################################
+
+"""
+cohere_core.utils
+=================
+
+This module returns available, balanced devices suited for given job.
+"""
+import os
+import ast
+import GPUtil
+from functools import reduce
+import cohere_core.utilities as ut
+
+
+__author__ = "Barbara Frosik"
+__copyright__ = "Copyright (c), UChicago Argonne, LLC."
+__docformat__ = 'restructuredtext en'
+__all__ = [
+           'estimate_no_proc',
+           'get_avail_gpu_runs',
+           'get_gpu_use',
+           'get_one_dev',
+           ]
+
+def estimate_no_proc(arr_size, factor):
+    """
+    Estimates number of processes the prep can be run on. Determined by number of available cpus and size
+    of array.
+    Parameters
+    ----------
+    arr_size : int
+        size of array
+    factor : int
+        an estimate of how much memory is required to process comparing to array size
+    Returns
+    -------
+    int
+        number of processes
+    """
+    from multiprocessing import cpu_count
+    import psutil
+
+    ncpu = cpu_count()
+    freemem = psutil.virtual_memory().available
+    nmem = freemem / (factor * arr_size)
+    # decide what limits, ncpu or nmem
+    if nmem > ncpu:
+        return ncpu
+    else:
+        return int(nmem)
+
+
+def get_avail_gpu_runs(devices, run_mem):
+    """
+    Finds how many jobs of run_mem size can run on configured GPUs on local host.
+
+    :param devices: list or string
+        list of GPU IDs or 'all' if configured to use all available GPUs
+    :param run_mem: int
+        size of GPU memory (in MB) needed for one job
+    :return: dict
+        pairs of GPU IDs, number of available jobs
+    """
+    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+    gpus = GPUtil.getGPUs()
+    available = {}
+
+    for gpu in gpus:
+        if devices == 'all' or gpu.id in devices:
+            available[gpu.id] = gpu.memoryFree // run_mem
+
+    return available
+
+
+def get_avail_hosts_gpu_runs(devices, run_mem):
+    """
+    This function is called in a cluster configuration case, i.e. devices parameter is configured as dictionary of hostnames and GPU IDs (either list of the IDs or 'all' for all GPUs per host).
+    It starts mpi subprocess that targets each of the configured host. The subprocess returns tuples with hostname and available GPUs. The tuples are converted into dictionary and returned.
+
+    :param devices:
+    :param run_mem:
+    :return:
+    """
+    hosts = ','.join(devices.keys())
+    script = ut.join(os.path.realpath(os.path.dirname(__file__)), 'host_utils.py')
+    command = ['mpiexec', '-n', str(len(devices)), '--host', hosts, 'python', script, str(devices), str(run_mem)]
+    result = subprocess.run(command, stdout=subprocess.PIPE, text=True).stdout
+    mem_map = {}
+    for entry in result.splitlines():
+        host_devs = ast.literal_eval(entry)
+        mem_map[host_devs[0]] = host_devs[1]
+    return mem_map
+
+
+def get_balanced_load(avail_runs, runs):
+    """
+    This function distributes the runs proportionally to the GPUs availability.
+    If number of available runs is less or equal to the requested runs, the input parameter avail_runs becomes load.
+    The function also returns number of available runs.
+
+    :param avail_runs: dict
+        keys are GPU IDs, and values are available runs
+        for cluster configuration the keys are prepended with the hostnames
+    :param runs: int
+        number of requested jobs
+    :return: dict, int
+        a dictionary with the same structure as avail_runs input parameter, but with values indicating runs modified to achieve balanced distribution.
+    """
+    if len(avail_runs) == 0:
+        return {}
+
+    # if total number of available runs is less or equal runs, return the avail_runs,
+    # and total number of available jobs
+    total_available = reduce((lambda x, y: x + y), avail_runs.values())
+    if total_available <= runs:
+        return avail_runs, total_available
+
+    # initialize variables for calculations
+    need_runs = runs
+    available = total_available
+    load = {}
+
+    # add one run from each available
+    for k, v in avail_runs.items():
+        if v > 0:
+            load[k] = 1
+            avail_runs[k] = v - 1
+            need_runs -= 1
+            if need_runs == 0:
+                return load, runs
+            available -= 1
+
+    # use proportionally from available
+    distributed = 0
+    ratio = need_runs / available
+    for k, v in avail_runs.items():
+        if v > 0:
+            share = int(v * ratio)
+            load[k] = load[k] + share
+            avail_runs[k] = v - share
+            distributed += share
+    need_runs -= distributed
+    available -= distributed
+
+    if need_runs > 0:
+        # need to add the few remaining
+        for k, v in avail_runs.items():
+            if v > 0:
+                load[k] = load[k] + 1
+                need_runs -= 1
+                if need_runs == 0:
+                    break
+
+    return load, runs
+
+
+def get_gpu_use(devices, no_jobs, job_size):
+    """
+    Determines available GPUs that match configured devices, and selects the optimal distribution of jobs on available devices. If devices is configured as dict (i.e. cluster configuration) then a file "hosts" is created in the running directory. This file contains hosts names and number of jobs to run on that host.
+    Parameters
+    ----------
+    devices : list or dict or 'all'
+        Configured parameter. list of GPU ids to use for jobs or 'all' if all GPUs should be used. If cluster configuration, then
+        it is dict with keys being host names.
+    no_jobs : int
+        wanted number of jobs
+    job_size : float
+        a GPU memory requirement to run one job
+    Returns
+    -------
+    picked_devs : list or list of lists(if cluster conf)
+        list of GPU ids that were selected for the jobs
+    available jobs : int
+        number of jobs allocated on all GPUs
+    cluster_conf : boolean
+        True is cluster configuration
+    """
+
+    def unpack_load(load):
+        picked_devs = []
+        for ds in [[k] * int(v) for k, v in load.items()]:
+            picked_devs.extend(ds)
+        return picked_devs
+
+    if type(devices) != dict:  # a configuration for local host
+        hostfile_name = None
+        avail_jobs = get_avail_gpu_runs(devices, job_size)
+        balanced_load, avail_jobs_no = get_balanced_load(avail_jobs, no_jobs)
+        picked_devs = unpack_load(balanced_load)
+    else:  # cluster configuration
+        hosts_avail_jobs = get_avail_hosts_gpu_runs(devices, job_size)
+        avail_jobs = {}
+        # collapse the host dict into one dict by adding hostname in front of key (gpu id)
+        for k, v in hosts_avail_jobs.items():
+            host_runs = {(f'{k}_{str(kv)}'): vv for kv, vv in v.items()}
+            avail_jobs.update(host_runs)
+        balanced_load, avail_jobs_no = get_balanced_load(avail_jobs, no_jobs)
+
+        # un-collapse the balanced load by hosts
+        host_balanced_load = {}
+        for k, v in balanced_load.items():
+            idx = k.rfind('_')
+            host = k[:idx]
+            if host not in host_balanced_load:
+                host_balanced_load[host] = {}
+            host_balanced_load[host].update({int(k[idx + 1:]): v})
+
+        # create hosts file and return corresponding picked devices
+        hosts_picked_devs = [(k, unpack_load(v)) for k, v in host_balanced_load.items()]
+
+        picked_devs = []
+        hostfile_name = f'hostfile_{os.getpid()}'
+        host_file = open(hostfile_name, mode='w+')
+        linesep = os.linesep
+        for h, ds in hosts_picked_devs:
+            host_file.write(f'{h}:{str(len(ds))}{linesep}')
+            picked_devs.append(ds)
+        host_file.close()
+
+    return picked_devs, int(min(avail_jobs_no, no_jobs)), hostfile_name
+
+
+def get_one_dev(ids):
+    """
+    Returns GPU ID that is included in the configuration, is on a local node, and has the most available memory.
+
+    :param ids: list or string or dict
+        list of gpu ids, or string 'all' indicating all GPUs included, or dict by hostname
+    :return: int
+        selected GPU ID
+    """
+    import socket
+
+    # if cluster configuration, look only at devices on local machine
+    if issubclass(type(ids), dict):  # a dict with cluster configuration
+        ids = ids[socket.gethostname()]  # configured devices on local host
+
+    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+    gpus = GPUtil.getGPUs()
+    dev = -1
+    max_mem = 0
+    # select one with the highest availbale memory
+    for gpu in gpus:
+        if ids == 'all' or gpu.id in ids:
+            free_mem = gpu.memoryFree
+            if free_mem > max_mem:
+                dev = gpu.id
+                max_mem = free_mem
+    return dev

cohere_ui/api/common.py

@@ -0,0 +1,131 @@
+# #########################################################################
+# Copyright (c) , UChicago Argonne, LLC. All rights reserved.             #
+#                                                                         #
+# See LICENSE file.                                                       #
+# #########################################################################
+
+import sys
+import os
+import cohere_core.utilities as ut
+import cohere_ui.api.convertconfig as conv
+
+
+def get_config_maps(experiment_dir, configs, **kwargs):
+    """
+    Reads the configuration files included in configs list and returns dictionaries.
+    It will check for missing main config, for converter version. If needed it will convert
+    to the latest version.
+
+    :param experiment_dir: str
+        directory where the experiment files are loacted
+    :param configs: list str
+        list of configuaration files key names requested by calling function
+        The main config is always processed.
+    :param kwargs: ver parameters
+        may contain:
+        - rec_id : reconstruction id, pointing to alternate config
+        - no_verify : boolean switch to determine if the verification error is returned
+    :return:
+        error message
+        configuration dictionaries
+        boolean value telling if conversion happened
+    """
+    no_verify = kwargs.pop('no_verify', False)
+    maps = {}
+    # always get main config
+    conf_dir = ut.join(experiment_dir, 'conf')
+    main_conf = ut.join(conf_dir, 'config')
+    if not os.path.isfile(main_conf):
+        # return 'no main config, exiting.', maps, None
+        raise ValueError('no main config, exiting.')
+    main_config_map = ut.read_config(main_conf)
+
+    msg = ut.verify('config', main_config_map)
+    if len(msg) > 0:
+        if not no_verify:
+            raise ValueError(msg)
+           # return msg, maps, None
+
+    converted = False
+
+    # convert configuration files if different converter version
+    if 'converter_ver' not in main_config_map or conv.get_version() is None or conv.get_version() > main_config_map['converter_ver']:
+        conv.convert(conf_dir)
+        main_config_map = ut.read_config(main_conf)
+        converted = True
+
+    maps['config'] = main_config_map
+
+    if 'config_instr' in configs or 'config_mp' in configs:
+        # the configuration file applies to specific beamline and needs to be imported
+        beamline = main_config_map.get('beamline', None)
+        if beamline is None:
+            raise ValueError(f'cannot import cohere_ui.beamlines.{beamline} module, exiting.')
+            # return f'cannot import cohere_ui.beamlines.{beamline} module, exiting.', maps, None
+        import importlib
+        beam_ver = importlib.import_module(f'cohere_beamlines.{beamline}.beam_verifier')
+    else:
+        beam_ver = None
+
+    verifier_map = {'config_data' : ut, 'config_rec' : ut, 'config_instr' : beam_ver,
+                    'config_prep' : ut, 'config_disp' : ut, 'config_mp' : beam_ver}
+
+    rec_id = kwargs.get('rec_id')
+    for conf in configs:
+        # special case for rec_id
+        if rec_id is not None and conf == 'config_rec':
+            conf_file = ut.join(experiment_dir, 'conf', f'{conf}_{rec_id}')
+        else:
+            conf_file = ut.join(experiment_dir, 'conf', conf)
+        if not os.path.isfile(conf_file):
+            continue
+        config_map = ut.read_config(conf_file)
+        # verify the config map, for beamline specific conf file the verifier has to be imported
+        msg = verifier_map[conf].verify(conf, config_map)
+        if len(msg) > 0:
+            if not no_verify:
+                raise ValueError(msg)
+
+        maps[conf] = config_map
+
+    return maps, converted
+
+
+def get_pkg(proc, dev):
+    pkg = 'np'
+
+    if proc == 'auto':
+        try:
+            import cupy
+            pkg = 'cp'
+            if dev == [-1]:
+                raise ValueError('cupy processing is available, define device')
+        except:
+            try:
+                import torch
+                pkg = 'torch'
+            except:
+                pass  # lib set to 'np'
+    elif proc == 'cp':
+        if sys.platform == 'darwin':
+            raise ValueError('cupy is not supported by Mac, select different processing')
+        try:
+            import cupy
+            if dev == [-1]:
+                raise ValueError('when using cupy processing, define device')
+            pkg = 'cp'
+        except:
+            raise ValueError('cupy is not installed, select different processing')
+    elif proc == 'torch':
+        try:
+            import torch
+            pkg = 'torch'
+        except:
+            raise ValueError('torch is not installed, select different processing')
+    elif proc == 'np':
+        pass  # lib set to 'np'
+    else:
+        err_msg = f'invalid "processing" value, {proc} is not supported'
+        raise ValueError(err_msg)
+
+    return pkg