mg-pso-gui 0.1.4__tar.gz → 0.1.6__tar.gz

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mg-pso-gui
-Version: 0.1.4
+Version: 0.1.6
 Summary: GUI for MG-PSO
 Author: Robert Cordingly
 Author-email: <rcording@uw.ed>

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/mg_pso_gui.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mg-pso-gui
-Version: 0.1.4
+Version: 0.1.6
 Summary: GUI for MG-PSO
 Author: Robert Cordingly
 Author-email: <rcording@uw.ed>

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/mg_pso_gui.egg-info/SOURCES.txt

@@ -40,4 +40,13 @@ mgpsogui/util/GraphGenerator.py
 mgpsogui/util/PSORunner.py
 mgpsogui/util/__init__.py
 mgpsogui/util/CTkToolTip/__init__.py
-mgpsogui/util/CTkToolTip/ctk_tooltip.py
+mgpsogui/util/CTkToolTip/ctk_tooltip.py
+mgpsogui/util/recosu/__init__.py
+mgpsogui/util/recosu/pso/__init__.py
+mgpsogui/util/recosu/pso/csip_access.py
+mgpsogui/util/recosu/pso/pso.py
+mgpsogui/util/recosu/utils/__init__.py
+mgpsogui/util/recosu/utils/trace_writer.py
+mgpsogui/util/recosu/utils/utils.py
+mgpsogui/util/recosu/utils/plot/__init__.py
+mgpsogui/util/recosu/utils/plot/cost_steps.py

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/mgpsogui/gui/HomePage.py

@@ -523,7 +523,7 @@ class App(customtkinter.CTk):
                         self.footer_progress_bar.start()
                         self.progress_message_left.configure(text="")
                         self.progress_message_middle.configure(text="Starting new round...")
-                        self.footer_progress_label.configure(text="Continuing...")
+                        self.footer_progress_label.configure(text="Next round...")
                         self.progress_message_right.configure(text="")
                 
                 

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/mgpsogui/util/PSORunner.py

@@ -7,19 +7,8 @@ import threading
 import time
 import os
 
-from cosu.pso import global_best
-
-class Unbuffered:
-    def __init__(self, stream):
-        self.stream = stream
-    def write(self, data):
-        self.stream.write(data)
-        self.stream.flush()
-    def writelines(self, datas):
-        self.stream.writelines(datas)
-        self.stream.flush()
-    def __getattr__(self, attr):
-        return getattr(self.stream, attr)
+#from cosu.pso import global_best
+from .recosu.pso import global_best
 
 def enqueue_output(out, queue):
     for line in iter(out.readline, b''):
@@ -52,8 +41,8 @@ def run_process(stdout_queue, stderr_queue, results_queue, data, folder):
     read_stdout, write_stdout = os.pipe()
     read_stderr, write_stderr = os.pipe()
     
-    sys.stdout = Unbuffered(os.fdopen(write_stdout, 'w'))
-    sys.stderr = Unbuffered(os.fdopen(write_stderr, 'w'))
+    sys.stdout = os.fdopen(write_stdout, 'w')
+    sys.stderr = os.fdopen(write_stderr, 'w')
     
     stdour_thread = threading.Thread(target=enqueue_output, args=(os.fdopen(read_stdout, 'r'), stdout_queue))
     stderr_thread = threading.Thread(target=enqueue_output, args=(os.fdopen(read_stderr, 'r'), stderr_queue))

mg-pso-gui-0.1.6/mgpsogui/util/recosu/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+__version__ = '1.2.8'
+__built__ = '1.2.8 955b3df883a5b6e63e360bf498da9f38cc0953a8 2022-01-27 10:14 -0700 od'
+__author__ = 'Olaf David'
+__email__ = 'odavid@colostate.edu'
+
+from .pso import pso
+__all__ = ["pso"]
+

mg-pso-gui-0.1.6/mgpsogui/util/recosu/pso/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+"""
+LUCA/PSO toolkit
+=========================================
+This is ...
+
+"""
+
+__author__ = """Olaf David"""
+__email__ = "odavid@colostate.edu"
+__version__ = "1.0"
+
+from .pso import global_best
+
+__all__ = ["global_best"]

mg-pso-gui-0.1.6/mgpsogui/util/recosu/pso/csip_access.py

@@ -0,0 +1,70 @@
+from cosu import utils
+from csip import Client
+from typing import List, Dict, Tuple
+import queue, os
+
+
+def csip_worker(reqq: queue.Queue, thread_no: int, stop, full_trace,
+                url, files, arg_params, conf: Dict, metainfo: Dict) -> None:
+    async_call = conf.get('async_call', True)  # default is async
+    save_resp = conf.get('save_response_to', None)  # save response, set it to a folder if responses should be saved.
+
+    while not stop():
+        try:
+            (rnd, step, iteration, particle, x, step_param_names, calib_params, objfunc, resq) = reqq.get(True, 0.5)
+            # print(thread_no, particle)
+
+            c = Client(metainfo=metainfo)
+
+            # static params (from args)
+            for param in arg_params:
+                c.add_data(param['name'], param['value'])
+
+            # particle params  (generated from steps)
+            # for i, value in enumerate(x):
+            for idx, value in enumerate(x[particle, :]):
+                c.add_data(step_param_names[idx], value)
+
+            # other, previously calibrated params (other steps)
+            for name, value in calib_params.items():
+                c.add_data(name, value)
+
+            # objective function info
+            for of in objfunc:
+                c.add_cosu(of['name'], of['of'], of['data'])
+                # c.add_data(of['name'], (of['data'][0], of['data'][1]))
+
+            print('.', end='', flush=True)
+
+            try:
+                # print(c)
+                if async_call:
+                    res = c.execute_async(url, files=files, conf=conf)
+                else:
+                    res = c.execute(url, files=files, conf=conf)
+
+                if res.is_failed():
+                    print(res)
+
+                if save_resp:
+                    res.save_to(os.path.join(save_resp, 'r{}s{}i{}p{}.json'.format(rnd, step, iteration, particle)))
+
+                # print(res)
+                print(u'\u2714', end='', flush=True)
+                cost = utils.calc_cost(res, objfunc)
+
+                if full_trace is not None:
+                    all_params = {}
+                    # for i, value in enumerate(x):
+                    for idx, value in enumerate(x[particle, :]):
+                        all_params[step_param_names[idx]] = value
+                    for name, value in calib_params.items():
+                        all_params[name] = value
+                    full_trace.append((all_params, cost))
+
+                resq.put((particle, cost))
+            except:
+                print(res)
+            reqq.task_done()
+        except queue.Empty:
+            continue

mg-pso-gui-0.1.6/mgpsogui/util/recosu/pso/pso.py

@@ -0,0 +1,326 @@
+#
+# $Id:$
+#
+# This file is part of the Cloud Services Integration Platform (CSIP),
+# a Model-as-a-Service framework, API, and application suite.
+#
+# 2012-2020, OMSLab, Colorado State University.
+#
+# OMSLab licenses this file to you under the MIT license.
+# See the LICENSE file in the project root for more information.
+#
+
+from ..utils import utils
+from .csip_access import csip_worker
+from pyswarms.single.global_best import GlobalBestPSO
+from os import path
+from threading import Thread
+from typing import Dict, List, Set, Tuple
+import numpy as np
+import copy
+import datetime
+import queue
+import json
+import os
+from multiprocessing import Queue
+
+
+def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_queue, files, url, param, conf: Dict, rnd,
+              step):
+    particles = len(x[:, 0])
+
+    pfail_count = conf.get('particles_fail', 1)  # Number of particles allowed to fail.
+    pfail_retry = conf.get('particles_retry', 3)  # retry number of times if more than allowed fail
+
+    while pfail_retry > 0:
+        cost = np.ones(particles)
+        res_queue = queue.Queue()
+
+        print('   ', end='', flush=True)
+
+        # submit for processing
+        # for i_particle, v in enumerate(x[:, 0]):
+        for particle in range(particles):
+            req_queue.put((rnd, step, iteration, particle, x, step_param_names, calib_params, step_objfunc, res_queue))
+            # req_queue.put((i_particle, x[i_particle,:], step_param_names, calib_params, step_objfunc, res_queue))
+
+        # wait for the cost value to come back
+        # for i, v in enumerate(x[:, 0]):
+        for idx in range(particles):
+            (particle, p_cost) = res_queue.get()
+            cost[particle] = p_cost
+            res_queue.task_done()
+
+        res_queue.join()
+
+        # replace the 'nan' cost values (failed/missing runs) with the mean of the
+        # rest of the cost values, hence ignore it
+
+        # print("cost ", cost)
+        nan_idx = np.where(np.isnan(cost))
+        failed_particles = len(nan_idx[0])
+
+        # leave the loop if fails acceptable
+        if failed_particles <= pfail_count:
+            break
+        print("Re-running particles, since ", failed_particles, ' out of ', particles, ' particles failed.')
+        pfail_retry -= 1
+
+    if pfail_retry == 0:
+        print('Particle evaluation failed ', conf.get('particles_retry', 3), ' times. PSO stopped.')
+        return None
+
+    # print("mean ", mean)
+    # assign the mean value to all failed runs.
+    mean = np.nanmean(cost)
+    cost[nan_idx[0]] = mean
+
+    print(flush=True)
+    return cost
+
+
+def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters: int, options: Dict,
+                oh_strategy: Dict = None, n_threads: int = 4, rtol: float = 0.001, ftol: float = -np.inf,
+                ftol_iter: int = 1, full_trace: List = None, rtol_iter: int = 1,
+                conf: Dict = None, metainfo: Dict = None, cost_target: float = -np.inf, result_queue: Queue = None) -> Tuple:
+    """Performs a stepwise particle swarm optimization PSO using a global best approach.
+
+        Parameters
+        ----------
+        steps : Dict
+            step definitions
+        rounds : tuple
+            round definition,  (min,max) or max
+        args : Dict
+            static service args
+        n_particles : int
+            number of particles
+        iters : int
+            number of iterations
+        options : Dict
+            PSO options (see pyswarms)
+        oh_strategy : Dict
+            PSO Option handling strategy (see pyswarms)
+        n_threads : int
+            size of thread pool (default: 4)
+        rtol : float
+            percentage of change of sum(best_cost) between rounds for
+            convergence. (Default is 0.001 0.1%)
+        ftol : float
+            PSO tolerance (default: -np.inf)
+        ftol_iter : float
+            number of iterations over which the relative error in
+            objective_func is acceptable for convergence. (default: 1)
+        full_trace : List
+            trace of all runs, list of tuples
+            first is dictionary of parameter names to parameter values
+            second is the cost value (default: None)
+        rtol_iter : int
+            the number of subsequent rounds with sum(best_cost) < rtol
+            (default: 1)
+        conf : Dict
+            configuration settings (default: {} )
+        metainfo : Dict
+             additional metainfo for the csip client (default: {} )
+        cost_target: float
+             the cost target (default: -np.inf)
+        Returns
+        -------
+        Tuple
+            optimizer: List, step_trace: Dict
+
+    """
+
+    utils.check_url(args['url'])
+
+    step_file = conf.get('step_trace', None)
+
+    min_rounds = 1
+    if type(rounds) == tuple:
+        min_rounds = rounds[0]
+        max_rounds = rounds[1]
+    else:
+        max_rounds = rounds
+
+    if min_rounds < 1:
+        raise Exception('min rounds >= 1 expected, was "{}"'.format(min_rounds))
+
+    if max_rounds > 20:
+        raise Exception('max rounds <= 20 expected, was "{}"'.format(max_rounds))
+
+    if n_threads < 1:
+        raise Exception('n_threads >= 1, was "{}"'.format(n_threads))
+
+    if rtol_iter < 1:
+        raise Exception('rtol_iter >= 1, was "{}"'.format(rtol_iter))
+
+    if full_trace is not None and not isinstance(full_trace, list):
+        raise Exception('full_trace must be of type, was "{}"'.format(type(full_trace)))
+
+    best_cost = np.ones(len(steps)) * np.inf
+    optimizer = np.empty(len(steps), dtype=object)
+
+    # trace of steps info
+    step_trace = {}
+
+    step_trace['dir'] = os.getcwd()
+    step_trace['start'] = str(datetime.datetime.now())
+    step_trace['min_rounds'] = min_rounds
+    step_trace['max_rounds'] = max_rounds
+    step_trace['iters'] = iters
+    step_trace['ftol'] = ftol
+    step_trace['ftol_iter'] = ftol_iter
+    step_trace['rtol'] = rtol
+    step_trace['rtol_iter'] = rtol_iter
+    step_trace['n_threads'] = n_threads
+    step_trace['n_particles'] = n_particles
+    step_trace['n_steps'] = len(steps)
+    step_trace['steps'] = copy.deepcopy(steps)
+    step_trace['args'] = args
+
+    if step_file is not None:
+        with open(step_file, "w") as fo:
+            json.dump(step_trace, fo)
+
+    # best round cost
+    best_round_cost = np.inf
+
+    # request queue for worker
+    req_queue = queue.Queue()
+
+    conf = conf or {}
+    done = False
+    thread_pool = []
+    for thread_no in range(n_threads):
+        worker = Thread(target=csip_worker, args=(req_queue, thread_no, lambda: done,
+                                                  full_trace, args['url'], args.get('files', None), args['param'],
+                                                  conf, metainfo)
+                        )
+        thread_pool.append(worker)
+        worker.start()
+
+    r_below = 0
+    early_exit = False
+    start_time = datetime.datetime.now()
+    for r in range(max_rounds):
+        no_improvement = np.full(len(steps), True)
+        for s, step in enumerate(steps):
+
+            # check if forced exit.
+            if path.exists("stop"):
+                print('\n>>>>> stop file found, exit now.')
+                early_exit = True
+                break
+
+            param_names, bounds, objfunc = utils.get_step_info(steps, s)
+            # maybe clone args?
+            args['step_param_names'] = param_names
+            args['step_objfunc'] = objfunc
+            # get calibrated parameter from all other steps
+            args['calib_params'] = utils.get_calibrated_params(steps, s)
+
+            args['req_queue'] = req_queue
+            args['conf'] = conf
+
+            # create optimizer in the first round.
+            if optimizer[s] is None:
+                optimizer[s] = GlobalBestPSO(step.get('n_particles', n_particles),
+                                             len(param_names),
+                                             oh_strategy=step.get('oh_strategy', oh_strategy),
+                                             options=step.get('options', options),
+                                             bounds=bounds,
+                                             ftol=step.get('ftol', ftol),
+                                             ftol_iter=step.get('ftol_iter', ftol_iter),
+                                             cost_target=step.get('cost_target', cost_target))
+            print('\n>>>>> R{}/S{}  particle params: {}  calibrated params: {}\n'.format(r + 1, s + 1, param_names,
+                                                                                         args['calib_params']))
+
+
+            if result_queue is not None:
+                result_queue.put((r + 1, s + 1, param_names, args['calib_params']))
+            
+            args['rnd'] = r + 1
+            args['step'] = s + 1
+
+            # perform optimization
+            cost, pos = optimizer[s].optimize(eval_cost, iters=step.get('iters', iters), **args)
+            if cost is None:
+                early_exit = True
+                break
+
+            # capture the best cost
+            # if cost < best_cost[s] and np.abs(cost - best_cost[s]) > rtol:
+            if cost < best_cost[s]:
+                best_cost[s] = cost
+                no_improvement[s] = False
+                utils.annotate_step(best_cost[s], pos, steps, s)
+
+            print('\n     Step summary, best particle values: {} '.format(pos))
+
+            key = "r{}s{}".format(r + 1, s + 1)
+            step_trace[key] = {}
+            step_trace[key]['time'] = str(datetime.datetime.now())
+            step_trace[key]['best_costs'] = best_cost
+            step_trace[key]['steps'] = copy.deepcopy(steps)
+
+            if step_file is not None:
+                with open(step_file, "w") as fo:
+                    json.dump(step_trace, fo)
+
+            # print(json.dumps(steps, sort_keys=False, indent=2))
+
+        if early_exit:
+            step_trace['exit'] = '1'
+            break
+
+        round_cost = np.sum(best_cost)
+
+        # if no improvement in all steps, break out of rounds prematurely
+        # but start checking only after min_rounds
+        # if (r + 1 >= min_rounds) and all(no_improvement):
+        rel_round_tol = 1 - round_cost / best_round_cost
+
+        print('\n  Round summary - round_cost:{}, step_costs: {}, step improvement:{}'
+              .format(round_cost, best_cost, np.invert(no_improvement)))
+        print('\n  Progress -  best_round_cost:{}, rel_round_tol:{}, rtol:{}'
+              .format(best_round_cost, rel_round_tol, rtol))
+
+        key = "r{}".format(r + 1)
+        step_trace[key] = {}
+        step_trace[key]['time'] = str(datetime.datetime.now())
+        step_trace[key]['round_cost'] = round_cost
+        step_trace[key]['best_costs'] = best_cost
+        step_trace[key]['improvements'] = no_improvement
+        if step_file is not None:
+            with open(step_file, "w") as fo:
+                json.dump(step_trace, fo)
+
+        if (r + 1 >= min_rounds) and 0 <= rel_round_tol < rtol:
+            r_below += 1
+            if r_below >= rtol_iter:
+                break
+        else:
+            # reset
+            r_below = 0
+
+        if round_cost < best_round_cost:
+            best_round_cost = round_cost
+
+    end_time = datetime.datetime.now()
+    elapsed = str(end_time - start_time)
+
+    print('Done in {} after {} out of {} rounds'.format(elapsed, r + 1, max_rounds))
+
+    done = True
+    for worker in thread_pool:
+        worker.join()
+
+    step_trace['rounds'] = r + 1
+    step_trace['end'] = str(datetime.datetime.now())
+    step_trace['time'] = elapsed
+
+    if step_file is not None:
+        with open(step_file, "w") as fo:
+            json.dump(step_trace, fo)
+
+    return optimizer, step_trace

mg-pso-gui-0.1.6/mgpsogui/util/recosu/utils/__init__.py

@@ -0,0 +1,10 @@
+from .utils import *
+
+
+__all__ = ["get_step_info",
+           "get_calibrated_params",
+           "annotate_step",
+           "NORM",
+           "check_url",
+           "save",
+           "load"]

mg-pso-gui-0.1.6/mgpsogui/util/recosu/utils/plot/__init__.py

@@ -0,0 +1,4 @@
+
+from .cost_steps import plot_cost_steps
+
+__all__ = ["plot_cost_steps"]

mg-pso-gui-0.1.6/mgpsogui/util/recosu/utils/plot/cost_steps.py

@@ -0,0 +1,111 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def rs_key(round, step):
+    return "r{}s{}".format(round, step)
+
+
+def get_values(name, s, steps):
+    nrounds = steps['rounds']
+    vals = []
+    for r in range(1, nrounds + 1):
+        info = steps[rs_key(r, s)]
+        params = info[s - 1]['param']
+        for i, p in enumerate(params):
+            if name == p['name']:
+                vals.append(p['value'])
+    return vals
+
+
+col = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown']
+
+
+def plot_cost_steps(optimizer, step_trace):
+    nsteps = step_trace['steps']
+    nrounds = step_trace['rounds']
+    iter = step_trace['iters']
+    particles = step_trace['particles']
+    time = step_trace['time']
+
+    total_iter = iter * nrounds
+
+    x = np.arange(1, total_iter + 1)
+    f = plt.figure(figsize=(15, 15))
+
+    # top plot
+    plt.subplot(nsteps + 1, 1, 1)
+
+    for i in range(nsteps):
+        plt.plot(x, optimizer[i].cost_history, label='step ' + str(i + 1), color=col[i + 2])
+
+    mini = optimizer[0].cost_history
+    for i in range(1, nsteps):
+        mini = np.maximum(mini, optimizer[i].cost_history)
+
+    for i, it in enumerate(range(iter, total_iter + 1, iter)):
+        plt.axvline(x=it, color='lightgray', linestyle='--')
+        plt.text(it, mini[0], ' R' + str(i + 1), color='lightgray')
+
+    plt.xlim(0, iter * nrounds)
+    plt.title('cost function\n (rounds:{} iter:{} particles:{} time:{})'.format(nrounds, iter, particles, time))
+    plt.legend()
+
+    px = np.arange(iter, (nrounds + 1) * iter, iter)
+    for s in range(1, nsteps + 1):
+
+        a = plt.subplot(nsteps + 1, 1, s + 1)
+
+        plt.subplots_adjust(hspace=0.3)
+        plt.xlim(0, iter * nrounds)
+        if s == nsteps:
+            plt.xlabel('Iterations')
+
+        # rounds marks
+        for i, it in enumerate(range(iter, total_iter + 1, iter)):
+            plt.axvline(x=it, color='lightgray', linestyle='--')
+
+        info = step_trace[rs_key(1, s)][s - 1]
+        # print(info['param'])
+        i = 0
+        params = info['param']
+
+        p = params[i]
+        # print('ax   ', plt)
+        vals = get_values(p['name'], s, step_trace)
+        title = "{}:{:.5f}".format(p['name'], vals[-1])
+        a.plot(px, vals, 'v', label=p['name'], color=col[i])
+        a.axhline(p['bounds'][0], linestyle='--', color='lightgray')
+        a.axhline(p['bounds'][1], linestyle='--', color='lightgray')
+        a.set_ylabel(p['name'], color=col[i])
+        a.tick_params(axis='y', labelcolor=col[i])
+        for x, y in zip(px, vals):
+            label = "{:.5f}".format(y)
+            a.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10),
+                       color="gray", ha='center')
+        a.legend(loc="center left")
+
+        if (len(params) > 1):
+            sp = a.twinx()
+            i = i + 1
+            p = params[i]
+            vals = get_values(p['name'], s, step_trace)
+            title = "{} {}:{:.5f}".format(title, p['name'], vals[-1])
+            # title = title + ', ' + p['name'] + ':' + str(vals[-1])
+            sp.plot(px, vals, 'v', label=p['name'], color=col[i])
+            sp.axhline(p['bounds'][0], linestyle='--', color='lightgray')
+            sp.axhline(p['bounds'][1], linestyle='--', color='lightgray')
+            sp.set_ylabel(p['name'], color=col[i])
+            sp.tick_params(axis='y', labelcolor=col[i])
+            for x, y in zip(px, vals):
+                label = "{:.5f}".format(y)
+                sp.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10),
+                            color="gray", ha='center')
+            sp.legend(loc="right")
+
+        # print(s, params)
+        plt.legend()
+        plt.title("step {}  ({})".format(s, title))
+
+    plt.xlabel('Iterations')
+    plt.show()

mg-pso-gui-0.1.6/mgpsogui/util/recosu/utils/trace_writer.py

@@ -0,0 +1,82 @@
+from typing import Dict, List, Set, Tuple
+
+TraceList = List[Tuple[Dict[str, float], float]]
+
+
+def write_run_trace(file: str, run_trace: TraceList, num_steps: int = 0, num_runs_per_step: int = 0) -> None:
+    if num_steps <= 0 or num_runs_per_step <= 0:
+        write_raw_run_trace(file, run_trace)
+        return
+
+    variable_list: List[str] = create_variable_list(run_trace)
+    split_trace_list: List[List[TraceList]] = split_trace(
+        run_trace, num_steps, num_runs_per_step)
+    try:
+        with open(file, "w") as writer:
+            writer.write("@T,trace\n")
+            writer.write("@H,cost,round,step,{}\n".format(",".join(variable_list)))
+            round: int = 1
+            for round_list in split_trace_list:
+                step: int = 1
+                for step_list in round_list:
+                    for param, cost in step_list:
+                        record = ",{},{},{}".format(cost, round, step)
+                        for name in variable_list:
+                            if name in param:
+                                record += ",{}".format(param[name])
+                            else:
+                                record += ","
+                        writer.write(record + "\n")
+                    step = step + 1
+                round = round + 1
+    except IOError:
+        print("Failed to write to file {}".format(file))
+
+
+def write_raw_run_trace(file: str, run_trace: TraceList) -> None:
+    variable_list: List[str] = create_variable_list(run_trace)
+    try:
+        with open(file, "w") as writer:
+            writer.write("@T,trace\n")
+            writer.write("@H,cost,{}\n".format(",".join(variable_list)))
+            for param, cost in run_trace:
+                record = ",{}".format(cost)
+                for name in variable_list:
+                    if name in param:
+                        record += ",{}".format(param[name])
+                    else:
+                        record += ","
+                writer.write(record + "\n")
+    except IOError:
+        print("Failed to write to file {}".format(file))
+
+
+def create_variable_list(run_trace: TraceList) -> List[str]:
+    variable_set: Set[str] = set()
+    for param, cost in run_trace:
+        for name in param.keys():
+            variable_set.add(name)
+    return list(variable_set)
+
+
+def split_trace(run_trace: TraceList, num_steps: int, num_runs_per_step: int) -> List[List[TraceList]]:
+    step = 1
+    i = 0
+    split_trace_list: List[List[TraceList]] = list()
+    step_list: List[TraceList]
+    while i < len(run_trace):
+        if step == 1:
+            step_list = list()
+            split_trace_list.append(step_list)
+
+        endpoint = i + num_runs_per_step
+        if endpoint > len(run_trace):
+            endpoint = len(run_trace)
+
+        step_list.append(run_trace[i:endpoint])
+        i = endpoint
+        step = step + 1
+        if step > num_steps:
+            step = 1
+
+    return split_trace_list

mg-pso-gui-0.1.6/mgpsogui/util/recosu/utils/utils.py

@@ -0,0 +1,137 @@
+#
+# $Id:$
+#
+# This file is part of the Cloud Services Integration Platform (CSIP),
+# a Model-as-a-Service framework, API, and application suite.
+#
+# 2012-2020, OMSLab, Colorado State University.
+#
+# OMSLab licenses this file to you under the MIT license.
+# See the LICENSE file in the project root for more information.
+#
+from builtins import Exception
+
+import requests
+import numpy as np
+import json
+from csip import Client
+
+NORM = {
+    'kge': lambda x: 1 - x,
+    'nslog': lambda x: 1 - x,
+    'nslog1p': lambda x: 1 - x,
+    'ns': lambda x: 1 - x,
+    'rmse': lambda x: x,
+    'trmse': lambda x: x,
+    'pbias': lambda x: abs(x)
+}
+
+
+def calc_cost(response: "Client", objfunc) -> float:
+    """aggregated objective function value -> cost"""
+
+    # this will eliminate the failed run as a candidate solution.
+    if not response.is_finished():
+        return np.nan
+
+    cost = 0.0
+    for o in objfunc:
+        cosu_val = response.get_cosu_value(o['name'])
+        if cosu_val is None:
+            raise Exception('No cosu of value for ' + o['name'])
+        cosu_of = o['of']
+        n = NORM.get(cosu_of, lambda x: x) # default
+        cost += n(cosu_val) * o.get('weight', 1.0)
+
+        # of_val = response.get_data_value(of['name'])
+        # for name in NORM:
+        #     if str(c['name']).startswith(name):
+        #         cost += NORM[name](cosu_val) * c.get('weight', 1.0)
+    return cost
+
+def save(file, dict):
+    """ Save dict to json file. """
+    with open(file, 'w') as json_file:
+        json.dump(dict, json_file, sort_keys=False, indent=2)
+
+
+def load(file):
+    """ Load dict from json file. """
+    with open(file) as json_file:
+        data = json.load(json_file)
+    return data
+
+
+def get_step_info(steps, index):
+    """Extract all relevant info from the step dict"""
+
+    step = steps[index]
+    l = len(step['param'])
+    param_min = np.ones(l)
+    param_max = np.ones(l)
+    param_names = []
+
+    # extract names and bounds
+    for i, p in enumerate(step['param']):
+        param_names.append(p['name'])
+        if len(p['bounds']) != 2:
+            raise Exception('Invalid bounds tuple: (min, max): "{}"'.format(p['bounds']))
+        if not p['bounds'][0] < p['bounds'][1]:
+            raise Exception('Invalid bounds values: "{}"'.format(p['bounds']))
+        param_min[i] = p['bounds'][0]
+        param_max[i] = p['bounds'][1]
+
+        # check if OF is supported
+    for o in step['objfunc']:
+        #  if not o['name'] in NORM:
+        #    raise Exception('OF not supported: "{}"'.format(o['name']))
+        found = False
+        for name in NORM:
+            if str(o['name']).startswith(name):
+                found = True
+        if not found:
+            raise Exception('OF not supported: "{}"'.format(o['name']))
+        if len(o['data']) != 2:
+            raise Exception('OF missing data: (sim, obs): "{}"'.format(o['name']))
+
+    return param_names, (param_min, param_max), step['objfunc']
+
+
+def get_calibrated_params(steps, index):
+    """Get all previously calibrated parameter from any other step"""
+
+    step = steps[index]
+    cp = {}
+    for s in steps:
+        # skip the own step
+        if s is step:
+            continue
+        for p in s['param']:
+            # if 'value' in p.keys():
+            if 'value' in p:
+                cp[p['name']] = p['value']
+
+    # if the step parameter are in any other step, take them out since
+    # we rather want to calibrate them
+    for p in step['param']:
+        if p['name'] in cp:
+            cp.pop(p['name'])
+
+    return cp
+
+
+def annotate_step(best_cost, pos, steps, index):
+    """Annotate the step with the best value"""
+
+    step = steps[index]
+    step['cost'] = best_cost
+    for i, p in enumerate(step['param']):
+        p['value'] = pos[i]
+
+
+def check_url(url):
+    """Check is the Url is valid."""
+
+    r = requests.head(url)
+    if r.status_code != 200:
+        raise Exception('Error code {} from Url: "{}"'.format(r.status_code, url))

{mg-pso-gui-0.1.4 → mg-pso-gui-0.1.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import codecs
 import os
 
-VERSION = '0.1.4'
+VERSION = '0.1.6'
 DESCRIPTION = 'GUI for MG-PSO'
 LONG_DESCRIPTION = open('../README.md').read()