mg-pso-gui 0.1.40__py3-none-any.whl → 0.2.75__py3-none-any.whl

mgpsogui/util/recosu/pso/pso.py

@@ -22,7 +22,7 @@ import datetime
 import queue
 import json
 import os
-from multiprocessing import Queue
+from multiprocessing import Queue as MPQueue
 
 
 def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_queue, files, url, param, conf: Dict, rnd,
@@ -82,7 +82,7 @@ def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_qu
 def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters: int, options: Dict,
                 oh_strategy: Dict = None, n_threads: int = 4, rtol: float = 0.001, ftol: float = -np.inf,
                 ftol_iter: int = 1, full_trace: List = None, rtol_iter: int = 1,
-                conf: Dict = None, metainfo: Dict = None, cost_target: float = -np.inf, result_queue: Queue = None) -> Tuple:
+                conf: Dict = None, metainfo: Dict = None, cost_target: float = -np.inf, result_queue: MPQueue = None) -> Tuple:
     """Performs a stepwise particle swarm optimization PSO using a global best approach.
 
         Parameters
@@ -168,7 +168,15 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
     step_trace['min_rounds'] = min_rounds
     step_trace['max_rounds'] = max_rounds
     step_trace['iters'] = iters
-    step_trace['ftol'] = ftol
+    
+    # BUG If ftol is -inf set it to a string
+    ftol_value = ftol
+    if ftol == -np.inf:
+        ftol_value = '-inf'
+    elif ftol == np.inf:
+        ftol_value = 'inf'
+    
+    step_trace['ftol'] = ftol_value
     step_trace['ftol_iter'] = ftol_iter
     step_trace['rtol'] = rtol
     step_trace['rtol_iter'] = rtol_iter
@@ -176,17 +184,21 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
     step_trace['n_particles'] = n_particles
     step_trace['n_steps'] = len(steps)
     step_trace['steps'] = copy.deepcopy(steps)
-    step_trace['args'] = args
+    step_trace['args'] = str(args) #BUG MUST BE REMOVED
 
     if step_file is not None:
         with open(step_file, "w") as fo:
             json.dump(step_trace, fo)
 
+    print("Wrote step trace")
+
     # best round cost
     best_round_cost = np.inf
 
     # request queue for worker
     req_queue = queue.Queue()
+    
+    print("Created queue")
 
     conf = conf or {}
     done = False
@@ -199,6 +211,8 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
         thread_pool.append(worker)
         worker.start()
 
+    print("Started worker threads")
+
     r_below = 0
     early_exit = False
     start_time = datetime.datetime.now()
@@ -222,6 +236,8 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
             args['req_queue'] = req_queue
             args['conf'] = conf
 
+            print("Calling global best..")
+
             # create optimizer in the first round.
             if optimizer[s] is None:
                 optimizer[s] = GlobalBestPSO(step.get('n_particles', n_particles),
@@ -232,21 +248,27 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
                                              ftol=step.get('ftol', ftol),
                                              ftol_iter=step.get('ftol_iter', ftol_iter),
                                              cost_target=step.get('cost_target', cost_target))
-            print('\n>>>>> R{}/S{}  particle params: {}  calibrated params: {}\n'.format(r + 1, s + 1, param_names,
-                                                                                         args['calib_params']))
+                
+            print('\n>>>>> R{}/S{}  particle params: {}  calibrated params: {}\n'.format(r + 1, s + 1, param_names, args['calib_params']))
 
-            if result_queue is not None:
-                result_queue.put('\n>>>>> R{}/S{}  particle params: {}  calibrated params: {}\n'.format(r + 1, s + 1, param_names, args['calib_params']))
+            #if result_queue is not None:
+            #    result_queue.put('\n>>>>> R{}/S{}  particle params: {}  calibrated params: {}\n'.format(r + 1, s + 1, param_names, args['calib_params']))
+            
+            print("Filled request queue...")
             
             args['rnd'] = r + 1
             args['step'] = s + 1
 
+            print("Evaluating cost...")
+
             # perform optimization
             cost, pos = optimizer[s].optimize(eval_cost, iters=step.get('iters', iters), **args)
             if cost is None:
                 early_exit = True
                 break
 
+            print("Finished evaluation...")
+
             # capture the best cost
             # if cost < best_cost[s] and np.abs(cost - best_cost[s]) > rtol:
             if cost < best_cost[s]:
@@ -262,7 +284,16 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
             key = "r{}s{}".format(r + 1, s + 1)
             step_trace[key] = {}
             step_trace[key]['time'] = str(datetime.datetime.now())
-            step_trace[key]['best_costs'] = best_cost
+            
+            best_costs_list = best_cost.tolist()
+            # If the cost is inf, set it to a string
+            for i, c in enumerate(best_costs_list):
+                if c == np.inf:
+                    best_costs_list[i] = 'inf'
+                elif c == -np.inf:
+                    best_costs_list[i] = '-inf'
+            
+            step_trace[key]['best_costs'] = best_costs_list # BUG
             step_trace[key]['steps'] = copy.deepcopy(steps)
 
             if step_file is not None:
@@ -299,8 +330,17 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
         step_trace[key] = {}
         step_trace[key]['time'] = str(datetime.datetime.now())
         step_trace[key]['round_cost'] = round_cost
-        step_trace[key]['best_costs'] = best_cost
-        step_trace[key]['improvements'] = no_improvement
+        
+        best_costs_list = best_cost.tolist() #BUG
+        # If the cost is inf, set it to a string
+        for i, c in enumerate(best_costs_list):
+            if c == np.inf:
+                best_costs_list[i] = 'inf'
+            elif c == -np.inf:
+                best_costs_list[i] = '-inf'
+        
+        step_trace[key]['best_costs'] = best_costs_list
+        step_trace[key]['improvements'] = no_improvement.tolist()
         if step_file is not None:
             with open(step_file, "w") as fo:
                 json.dump(step_trace, fo)
@@ -336,4 +376,8 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
         with open(step_file, "w") as fo:
             json.dump(step_trace, fo)
 
+    if result_queue is not None:
+        result_queue.put("Step Trace")
+        result_queue.put(step_trace)
+
     return optimizer, step_trace

mgpsogui/util/recosu/sampling/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+"""
+LUCA/PSO toolkit
+=========================================
+This is ...
+
+"""
+
+__author__ = """Olaf David"""
+__email__ = "odavid@colostate.edu"
+__version__ = "1.0"
+
+from .sampling import run_sampler
+
+__all__ = ["run_sampler"]

mgpsogui/util/recosu/sampling/halton/halton.py

@@ -0,0 +1,45 @@
+from collections.abc import Iterable
+import math
+from ...sampling.halton.prime import generate_n_primes
+
+
+def halton(index: int, base: int) -> float:
+    fraction: float = 1.0
+    result: float = 0
+
+    while index > 0:
+        fraction = fraction / base
+        result += fraction * (index % base)
+        index = math.floor(index / base)
+
+    return result
+
+
+class HaltonSampleGenerator:
+    index: int
+    maxIndex: int
+    primes: list[int]
+
+    def __init__(self, count: int, offset: int, num_parameters: int):
+        assert (count > 0)
+        assert (offset >= 0)
+        self.index = offset + 1
+        self.maxIndex = offset + count + 1
+        self.primes = generate_n_primes(num_parameters)
+
+    def __iter__(self) -> Iterable[tuple[int, list[float]]]:
+        return self
+
+    def __next__(self) -> tuple[int, list[float]]:
+        if self.index >= self.maxIndex:
+            raise StopIteration
+
+        i = 0
+        values: list[float] = []
+        for base in self.primes:
+            values.append(halton(self.index, base))
+
+        result: tuple[int, list[float]] = (self.index, values)
+        self.index = self.index + 1
+        return result
+

mgpsogui/util/recosu/sampling/halton/prime.py

@@ -0,0 +1,82 @@
+from typing import List
+
+PRIME_TABLE: List[int] = [
+    2, 3, 5, 7, 11, 13, 17, 19, 23, 29,
+    31, 37, 41, 43, 47, 53, 59, 61, 67, 71,
+    73, 79, 83, 89, 97, 101, 103, 107, 109, 113,
+    127, 131, 137, 139, 149, 151, 157, 163, 167, 173,
+    179, 181, 191, 193, 197, 199, 211, 223, 227, 229,
+    233, 239, 241, 251, 257, 263, 269, 271, 277, 281,
+    283, 293, 307, 311, 313, 317, 331, 337, 347, 349,
+    353, 359, 367, 373, 379, 383, 389, 397, 401, 409,
+    419, 421, 431, 433, 439, 443, 449, 457, 461, 463,
+    467, 479, 487, 491, 499, 503, 509, 521, 523, 541,
+    547, 557, 563, 569, 571, 577, 587, 593, 599, 601,
+    607, 613, 617, 619, 631, 641, 643, 647, 653, 659,
+    661, 673, 677, 683, 691, 701, 709, 719, 727, 733,
+    739, 743, 751, 757, 761, 769, 773, 787, 797, 809,
+    811, 821, 823, 827, 829, 839, 853, 857, 859, 863,
+    877, 881, 883, 887, 907, 911, 919, 929, 937, 941,
+    947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013,
+    1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069,
+    1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151,
+    1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223,
+    1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291,
+    1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373,
+    1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451,
+    1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511,
+    1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583,
+    1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657,
+    1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733,
+    1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811,
+    1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889,
+    1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987,
+    1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053,
+    2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129,
+    2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213,
+    2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287,
+    2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357,
+    2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423,
+    2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531,
+    2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617,
+    2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687,
+    2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741,
+    2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819,
+    2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903,
+    2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999,
+    3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079,
+    3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181,
+    3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257,
+    3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331,
+    3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413,
+    3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511,
+    3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571,
+    3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643,
+    3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727,
+    3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821,
+    3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907,
+    3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989,
+    4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057,
+    4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139,
+    4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231,
+    4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297,
+    4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409
+]
+
+
+def generate_n_primes(n: int) -> List[int]:
+    if n < len(PRIME_TABLE):
+        return PRIME_TABLE[0:n]
+
+    primes: List[int] = PRIME_TABLE.copy()
+    num: int = primes[-1] + 2
+    while len(primes) < n:
+        is_prime = True
+        for p in primes:
+            if num % p == 0:
+                is_prime = False
+                break
+        if is_prime:
+            primes.append(num)
+        num += 2
+    return primes

mgpsogui/util/recosu/sampling/random/random_sampler.py

@@ -0,0 +1,34 @@
+from collections.abc import Iterable
+import random
+
+
+class RandomSampleGenerator:
+    index: int
+    count: int
+    num_parameters: int
+    rand: random.Random
+
+    def __init__(self, count: int, num_parameters: int):
+        assert (count > 0)
+        self.index = 1
+        self.count = count + 1
+        self.num_parameters = num_parameters
+        self.rand = random.Random()
+
+    def __iter__(self) -> Iterable[tuple[int, list[float]]]:
+        return self
+
+    def __next__(self) -> tuple[int, list[float]]:
+        if self.index >= self.count:
+            raise StopIteration
+
+        i: int = 0
+        values: list[float] = []
+        while i < self.num_parameters:
+            values.append(self.rand.uniform(0, 1))
+            i = i + 1
+
+        result: tuple[int, list[float]] = (self.index, values)
+        self.index = self.index + 1
+        return result
+

mgpsogui/util/recosu/sampling/sample_trace_writer.py

@@ -0,0 +1,47 @@
+import threading
+
+
+class SampleTraceWriter:
+    trace_file: str
+    parameter_indices: dict[int, str]
+    objective_indices: dict[int, str]
+    write_lock: threading.Lock
+
+    def __init__(self, trace_file: str):
+        assert(trace_file is not None and len(trace_file) > 0)
+        self.trace_file = trace_file
+        self.parameter_indices = {}
+        self.objective_indices = {}
+        self.write_lock = threading.Lock()
+
+    def write_header(self, parameter_names: list[str], objective_names: list[str]) -> None:
+        with self.write_lock:
+            with open(self.trace_file, 'w') as writer:
+                writer.write("id")
+                self.parameter_indices = {}
+                index: int = 0
+                for name in parameter_names:
+                    writer.write(",{}".format(name))
+                    self.parameter_indices[index] = name
+                    index = index + 1
+                self.objective_indices = {}
+                index = 0
+                for name in objective_names:
+                    writer.write(",{}".format(name))
+                    self.objective_indices[index] = name
+                    index = index + 1
+                writer.write("\n")
+
+    def append_sample(self, sample_id: int, parameters: dict[str, any], objectives: dict[str, any]) -> None:
+        with self.write_lock:
+            with open(self.trace_file, 'a') as writer:
+                writer.write("{}".format(sample_id))
+                index: int = 0
+                while index < len(self.parameter_indices):
+                    writer.write(",{}".format(parameters[self.parameter_indices[index]]))
+                    index = index + 1
+                index = 0
+                while index < len(self.objective_indices):
+                    writer.write(",{}".format(objectives[self.objective_indices[index]]))
+                    index = index + 1
+                writer.write("\n")

mgpsogui/util/recosu/sampling/sampler_task.py

@@ -0,0 +1,75 @@
+import os
+from csip import Client
+
+
+class SamplerTask:
+    task_id: int
+    parameters: dict[str, any]
+    objectives: list[dict[str, any]]
+    static_parameters: dict[str, any]
+    url: str
+    files: list[str]
+    metainfo: dict[str, any]
+    conf: dict[str, any]
+    result: dict[str, any]
+
+    def __init__(self, task_id: int, parameters: dict[str, any], objectives: list[dict[str, any]],
+                 static_parameters: dict[str, any], url: str, files: list[str] = None, metainfo: dict[str, any] = None,
+                 conf: dict[str, any] = None):
+        self.task_id = task_id
+        assert (parameters is not None and len(parameters) > 0)
+        self.parameters = parameters
+        assert (objectives is not None and len(objectives) > 0)
+        self.objectives = objectives
+        self.static_parameters = static_parameters if static_parameters is not None else []
+        assert (url is not None and len(url) > 0)
+        self.url = url
+        self.files = files if files is not None else []
+        self.metainfo = metainfo
+        self.conf = conf
+
+    def create_request(self) -> Client:
+        request: Client = Client(metainfo=self.metainfo)
+
+        for key, value in self.static_parameters.items():
+            request.add_data(key, value)
+
+        for key, value in self.parameters.items():
+            request.add_data(key, value)
+
+        for of in self.objectives:
+            request.add_cosu(of['name'], of['of'], of['data'])
+
+        return request
+
+    def run_task(self) -> bool:
+        self.result = {}
+        request: Client = self.create_request()
+        async_call: bool = self.conf.get('async_call', True) if self.conf is not None else True
+        # save response, set it to a folder if responses should be saved.
+        save_resp = self.conf.get('save_response_to', None) if self.conf is not None else None
+        successful: bool = False
+
+        response: Client = None
+        try:
+            if async_call:
+                response = request.execute_async(self.url, files=self.files, conf=self.conf)
+            else:
+                response = request.execute(self.url, files=self.files, conf=self.conf)
+
+            successful = response.is_finished()
+            if not successful:
+                print(response)
+
+            if save_resp:
+                response.save_to(os.path.join(save_resp, 'task_{}.json'.format(self.task_id)))
+
+            objectives: list[dict[str, str]] = response.get_metainfo("cosu")
+            for of in objectives:
+                self.result[of["name"]] = of["value"]
+        except Exception as ex:
+            print(ex)
+            print(response)
+            successful = False
+
+        return successful

mgpsogui/util/recosu/sampling/sampling.py

@@ -0,0 +1,99 @@
+from collections.abc import Iterable
+import math
+import asyncio
+import concurrent
+import datetime
+from ..utils import utils
+from ..sampling.halton.halton import HaltonSampleGenerator
+from ..sampling.random.random_sampler import RandomSampleGenerator
+from ..sampling.sampler_task import SamplerTask
+from ..sampling.sample_trace_writer import SampleTraceWriter
+
+
+def weighted_value(weight: float, lower: float, upper: float) -> float:
+    return lower + weight * (upper - lower)
+
+
+def get_static_parameters(args: dict[str, any]) -> dict[str, any]:
+    static_parameters: dict[str, any] = {}
+    for param in args["param"]:
+        static_parameters[param["name"]] = param["value"]
+    return static_parameters
+
+
+def get_objective_names(objfunc: dict[str, any]) -> list[str]:
+    objective_names: list[str] = []
+    for of in objfunc:
+        objective_names.append(of["name"])
+    return objective_names
+
+
+def thread_function(task: SamplerTask) -> tuple[bool, SamplerTask]:
+    return task.run_task(), task
+
+
+def create_generator(method: str, count: int, num_parameters: int, **kwargs) -> Iterable[tuple[int, list[float]]]:
+    if method == "halton":
+        offset: int = 0
+        if "offset" in kwargs:
+            offset = kwargs["offset"]
+        return HaltonSampleGenerator(count, offset, num_parameters)
+    elif method == "random":
+        return RandomSampleGenerator(count, num_parameters)
+
+    raise Exception("Sampling method is not recognized")
+
+
+def run_sampler(steps: list[dict[str, any]], args: dict[str, any], count: int, num_threads: int, method: str = "halton",
+                metainfo: dict[str, any] = None, conf: dict[str, any] = None, trace_file: str = "trace.csv",
+                **kwargs) -> dict[int, tuple[dict[str, any], dict[str, any]]]:
+    param_names, bounds, objfunc = utils.get_step_info(steps, 0)
+    generator: Iterable[tuple[int, list[float]]] = create_generator(method, count, len(param_names), **kwargs)
+    objective_names: list[str] = get_objective_names(objfunc)
+    static_parameters: dict[str, any] = get_static_parameters(args)
+    url: str = args["url"]
+    files: list[str] = args["files"]
+
+    trace: dict[int, tuple[dict[str, float], dict[str, float]]] = {}
+    trace_writer: SampleTraceWriter = SampleTraceWriter(trace_file)
+    trace_writer.write_header(param_names, objective_names)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        futures = []
+        for sample_id, sample in generator:
+            params: dict[str, float] = {}
+            index: int = 0
+            while index < len(sample):
+                params[param_names[index]] = weighted_value(sample[index], bounds[0][index], bounds[1][index])
+                index += 1
+
+            task: SamplerTask = SamplerTask(sample_id, params, objfunc, static_parameters, url, files, metainfo, conf)
+            futures.append(executor.submit(thread_function, task))
+        # for future in concurrent.futures.as_completed(futures):
+        #     pass
+        num_finished: int = 0
+        percentage: float
+        last_percentage: float = 0
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                successful, task = future.result()
+
+                if successful:
+                    trace[task.task_id] = (task.parameters, task.result)
+                    trace_writer.append_sample(task.task_id, task.parameters, task.result)
+                else:
+                    print("Failed to successfully execute task: {}", task.task_id, flush=True)
+            except asyncio.CancelledError as ce:
+                pass
+            except asyncio.InvalidStateError as ise:
+                pass
+            except Exception as ex:
+                print(ex, flush=True)
+
+            num_finished = num_finished + 1
+            percentage = math.trunc(num_finished / count * 1000) / 10
+            if percentage > last_percentage:
+                last_percentage = percentage
+                print("{}% Done {}".format(percentage, datetime.datetime.now()), flush=True)
+
+    return trace