mg-pso-gui 0.1.12__py3-none-any.whl → 0.1.40__py3-none-any.whl

mgpsogui/gui/RunTab/RunTab.py

@@ -19,21 +19,7 @@ def create_tab(self, tab):
     tab.grid_columnconfigure(0, weight=1)
     tab.grid_rowconfigure(0, weight=1)
     tab.grid_rowconfigure(1, weight=200)
-    
-    """
-    self.url = customtkinter.CTkEntry(tab, textvariable=self.option_manager.get_arguments()['url'])
-    self.url.grid(row=0, column=0, columnspan=1, padx=(20, 20), pady=(20, 20), sticky="nsew")
 
-    self.run_button = customtkinter.CTkButton(tab, text="Run", command=self.run)
-    self.run_button.grid(row=0, column=1, padx=(20, 20), pady=(20, 20), sticky="nsew")
-    ctt(self.run_button, delay=0.5, message="Start calibration...")
-    
-        self.stop_button = customtkinter.CTkButton(tab, text="Stop", command=self.stop)
-    self.stop_button.grid(row=0, column=2, padx=(20, 20), pady=(20, 20), sticky="nsew")
-    ctt(self.stop_button, delay=0.5, message="Stop calibration...")
-    
-    """
-    
     self.progress_container = customtkinter.CTkFrame(tab)
     self.progress_container.grid_columnconfigure(0, weight=1)
     self.progress_container.grid_columnconfigure(1, weight=1)
@@ -50,12 +36,7 @@ def create_tab(self, tab):
     self.progress_message_right = customtkinter.CTkLabel(self.progress_container, text="")
     self.progress_message_right.grid(row=0, column=2, padx=(10, 10), pady=(10, 10), sticky="e")
     
-    #self.progress_bar = customtkinter.CTkProgressBar(self.progress_container)
-    #self.progress_bar.grid(row=1, column=0, columnspan=3, padx=(10, 10), pady=(10, 10), sticky="ew")
-    #self.progress_bar.set(0)
-    #ctt(self.progress_bar, delay=0.5, message="Current calibration progress")
-    
     self.textbox = customtkinter.CTkTextbox(tab)
     self.textbox.grid(row=1, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
-    self.textbox.insert("0.0", "Welcome to the CSIP PSO Calibration Tool!\n\nUse the Calibration tab to define steps and calibration parameters. Use this tab to run and observe calibration progress. Once finished, use the Visualization tab to generate figures and graphs.")
+    self.textbox.insert("0.0", "Welcome to the CSIP PSO Calibration Tool!\n\nUse the Setup tab to define steps and calibration parameters. Use this tab to view logs and observe calibration progress. Once finished, use the Results tab to generate figures and graphs.")
         

mgpsogui/gui/VisualizeTab/SideBar.py

@@ -0,0 +1,108 @@
+from customtkinter import CTkScrollableFrame
+from customtkinter import CTkFrame
+from customtkinter import CTkLabel
+from customtkinter import CTkButton
+from customtkinter import CTkEntry
+from customtkinter import CTkTextbox
+from customtkinter import CTkImage
+from customtkinter import CTkOptionMenu
+from PIL import Image
+import os
+
+import pandas as pd
+
+class SideBar(CTkScrollableFrame):
+    def __init__(self, *args,
+                 option_manager: None,
+                 home_page: None,
+                 **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.option_manager = option_manager
+        self.home_page = home_page
+        
+        self.render()
+
+    def clear(self):
+        self.containerFrame.destroy()
+
+    def refresh(self):
+        self.clear()
+        self.render()
+
+    def render(self):
+
+        self.containerFrame = CTkFrame(self, width=300, fg_color="transparent")
+        self.containerFrame.grid(row=0, column=0, padx=(
+            0, 0), pady=(0, 0), sticky="ew")
+        self.containerFrame.grid_columnconfigure(0, weight=1)
+        
+        selected_graph = self.home_page.graph_selector_value.get()
+        
+        if (selected_graph == "Best Cost Stacked"):
+            self.graph_label = CTkLabel(self.containerFrame, text="Best Cost Stacked")
+            self.graph_label.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+            pass
+        elif (selected_graph == "Best Cost by Round"):
+            self.graph_label = CTkLabel(self.containerFrame, text="Best Cost by Round")
+            self.graph_label.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+            pass
+        elif (selected_graph == "Iteration Table"):
+            self.graph_label = CTkLabel(self.containerFrame, text="Iteration Table")
+            self.graph_label.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+            pass
+        elif (selected_graph == "Calibrated Parameters"):
+            self.graph_label = CTkLabel(self.containerFrame, text="Calibrated Parameters")
+            self.graph_label.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+        elif (selected_graph == "Custom CSV"):
+            
+            info = self.option_manager.get_project_data()
+            folder = os.path.join(info['path'], info['name'])
+            if not os.path.exists(folder):
+                os.makedirs(folder)
+                
+            # Get all CSV files in the folder and add their paths to a list
+            path_map = {}
+            name_list = []
+            for root, dirs, files in os.walk(folder):
+                for file in files:
+                    if file.endswith(".csv"):
+                        name = file.replace(".csv", "")
+                        name_list.append(name)
+                        path_map[name] = os.path.join(root, file)
+            
+            if (len(name_list) == 0):
+                name_list.append("No files found...")
+            else:
+                if (self.home_page.selected_csv.get() not in name_list):
+                    self.home_page.selected_csv.set(name_list[0])
+            
+            self.home_page.csv_file_selector = CTkOptionMenu(self.containerFrame, values=name_list, variable=self.home_page.selected_csv, command=self.home_page.update_graph)
+            self.home_page.csv_file_selector.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+            
+            selected_file = self.home_page.selected_csv.get()
+            if (selected_file in path_map and selected_file != self.home_page.open_file):
+                self.home_page.csv_data = pd.read_csv(path_map[selected_file], skiprows=3)
+                self.home_page.open_file = selected_file
+                
+            if (self.home_page.csv_data != None):
+                # Get all column names of CSV
+                columns = self.home_page.csv_data.columns
+                
+                self.home_page.csv_x_selector = CTkOptionMenu(self.containerFrame, values=columns, variable=self.home_page.selected_x, command=self.home_page.update_graph)
+                self.home_page.csv_x_selector.grid(row=2, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+                
+                if (self.home_page.csv_x_selector.get() not in columns):
+                    self.home_page.csv_x_selector.set(columns[1])
+                
+                self.home_page.csv_y1_selector = CTkOptionMenu(self.containerFrame, values=columns, variable=self.home_page.selected_y1, command=self.home_page.update_graph)
+                self.home_page.csv_y1_selector.grid(row=2, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+                
+                if (self.home_page.csv_y1_selector.get() not in columns):
+                    self.home_page.csv_y1_selector.set(columns[2])
+                
+                self.home_page.csv_y2_selector = CTkOptionMenu(self.containerFrame, values=columns, variable=self.home_page.selected_y2, command=self.home_page.update_graph)
+                self.home_page.csv_y2_selector.grid(row=2, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
+                
+                if (self.home_page.csv_y2_selector.get() not in columns):
+                    self.home_page.csv_y2_selector.set(columns[3])

mgpsogui/gui/VisualizeTab/VisualizeTab.py

@@ -5,6 +5,8 @@ import os
 import platform
 import subprocess
 
+from . import SideBar
+
 def create_tab(self, tab):
     
     def open_graph_in_browser():
@@ -45,7 +47,7 @@ def create_tab(self, tab):
         self.graph_label.update_idletasks()
     
     
-    tab.grid_columnconfigure(0, weight=1)
+    tab.grid_columnconfigure(0, weight=2)
     tab.grid_columnconfigure(1, weight=8)
     tab.grid_rowconfigure(0, weight=1)
     
@@ -55,15 +57,17 @@ def create_tab(self, tab):
     self.graph_container.grid(row=0, column=1, padx=(20, 20), pady=(20, 20), sticky="nsew")
     
     self.graph_sidebar.grid_columnconfigure(0, weight=1)
-    self.graph_sidebar.grid_rowconfigure(8, weight=1)
+    self.graph_sidebar.grid_rowconfigure(1, weight=1)
     self.graph_container.grid_columnconfigure(0, weight=1)
     self.graph_container.grid_rowconfigure(0, weight=1)
     
-    self.graph_selector_value = tk.StringVar()
-    self.graph_selector_value.set("Best Cost Stacked")
-    self.graph_selector = customtkinter.CTkOptionMenu(self.graph_sidebar, values=["Best Cost Stacked", "Best Cost by Round", "Calibrated Parameters", "Iteration Table"], variable=self.graph_selector_value, command=self.update_graph)
+    self.graph_selector = customtkinter.CTkOptionMenu(self.graph_sidebar, values=["Best Cost Stacked", "Best Cost by Round", "Calibrated Parameters", "Iteration Table", "Custom CSV"], variable=self.graph_selector_value, command=self.update_graph)
     self.graph_selector.grid(row=0, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")
     
+    # Create SideBar
+    self.vis_sidebar = SideBar.SideBar(self.graph_sidebar, option_manager=self.option_manager, home_page=self, fg_color="transparent")
+    self.vis_sidebar.grid(row=1, column=0, rowspan=8, padx=(0, 0), pady=(0, 0), sticky="nsew")
+    
     # Add a button to call open_graph_in_browser
     self.graph_button = customtkinter.CTkButton(self.graph_sidebar, text="Open in Browser", command=open_graph_in_browser)
     self.graph_button.grid(row=9, column=0, padx=(20, 20), pady=(20, 20), sticky="nsew")

mgpsogui/util/GraphGenerator.py

@@ -3,6 +3,40 @@ import plotly.graph_objs as go
 import pandas as pd
 import numpy as np
 import os
+from PIL import Image, ImageTk
+import customtkinter
+
+def generate_graphs(HomePage):
+    try:
+        selected_graph = HomePage.graph_selector_value.get()
+        info = HomePage.option_manager.get_project_data()
+        folder = os.path.join(info['path'], info['name'])
+        if not os.path.exists(folder):
+            os.makedirs(folder)
+            
+        if (selected_graph == "Best Cost Stacked"):
+            HomePage.selected_graph_name = "best_cost_stacked"
+            best_cost_stacked(HomePage.running_config['steps'], HomePage.progress_data, HomePage.option_manager)
+        elif (selected_graph == "Best Cost by Round"):
+            HomePage.selected_graph_name = "best_cost_by_round"
+            best_cost_by_round(HomePage.running_config['steps'], HomePage.progress_data, HomePage.option_manager)
+        elif (selected_graph == "Iteration Table"):
+            HomePage.selected_graph_name = "table"
+            table(HomePage.running_config['steps'], HomePage.progress_data, HomePage.option_manager)
+        elif (selected_graph == "Calibrated Parameters"):
+            HomePage.selected_graph_name = "calibrated_params_by_round"
+            calibrated_params_by_round(HomePage.running_config['steps'], HomePage.calibration_data, HomePage.option_manager)
+            
+        image_path = os.path.join(folder, HomePage.selected_graph_name + ".png")
+        
+        if not os.path.exists(image_path):
+            image_path = os.path.join("./images", "up.png")
+        
+        HomePage.graph_image_obj = Image.open(image_path)
+        HomePage.graph_image = customtkinter.CTkImage(HomePage.graph_image_obj, size=(HomePage.image_width * HomePage.image_scale, HomePage.image_height * HomePage.image_scale))
+        HomePage.graph_label.configure(image=HomePage.graph_image)
+    except Exception as e:
+        pass
 
 def best_cost_stacked(config, dataframe, option_manager):
     fig = go.Figure()

mgpsogui/util/PSORunner.py

@@ -1,4 +1,5 @@
 import csip
+from csip import Client
 import cosu
 import sys
 from multiprocessing import Process, Queue
@@ -99,81 +100,43 @@ def run_process(stdout_queue, stderr_queue, results_queue, cosu_queue, data, fol
     sys.stderr = old_stderr
     results_queue.put((optimizer, trace))
 
+def get_results():
+    request: Client = Client()
+    for name, value in parameters.items():
+        # if parameter name has a / in it assume that is a file based parameter and therefore value needs to be an array
+        if "/" in name and type(value) is not list:
+            request.add_data(name, [value])
+        else:
+            request.add_data(name, value)
+
+    conf = {
+        'service_timeout': 60.0 # (sec)
+    }
+    files: List[str] = [] #optional list of filenames
+
+    #Synchronous Call
+    result: Client = request.execute(CSIP_ENDPOINT, files=files, sync=True, conf=conf)
+
+    #Asynchronous Call
+    tsamp: float = 0
+    def callback(c: Client, progress: str):
+        tsamp2: float = time.time()
+        print('Halton Update {} - {} - {}'.format(halton_id, c.get_status(), tsamp2 - tsamp))
+
+
+    tsamp = time.time()
+    result: Client = request.execute_async(
+        CSIP_ENDPOINT,
+        files=files,
+        callback=callback,
+        first_poll=poll_time,
+        next_poll=poll_time,
+        conf=conf
+    )
+    # After recieving response
+    if result.is_finished():
+        print(result)
+    else:
+        print(result)
 
 
-
-"""import csip
-import cosu
-import sys
-import multiprocessing
-import threading
-import time
-import os
-
-from cosu.pso import global_best
-
-def run_process(process_queue, data, folder):
-    steps = data['steps']
-    args = data['arguments']
-    calib = data['calibration_parameters']
-    
-    calibration_map = {}
-    for param in calib:
-        param_name = param['name']
-        param_value = param['value']
-        calibration_map[param_name] = param_value
-        
-    if not os.path.exists(folder):
-        os.makedirs(folder)
-
-    if (os.path.exists(os.path.join(folder, 'output.txt'))):
-        os.remove(os.path.join(folder, 'output.txt'))
-        
-    if (os.path.exists(os.path.join(folder, 'error.txt'))):
-        os.remove(os.path.join(folder, 'error.txt'))
-
-    sys.stdout = open(os.path.join(folder, 'output.txt'), 'w', buffering=1)
-    sys.stderr = open(os.path.join(folder, 'error.txt'), 'w', buffering=1)
-    
-    options = {}
-    oh_strategy = {}
-    
-    for key in calibration_map.keys():
-        if "options_" in key:
-            options[key.replace("options_", "")] = float(calibration_map[key])
-        if "strategy_" in key:
-            oh_strategy[key.replace("strategy_", "")] = calibration_map[key]
-
-    print("\n")
-    print(calibration_map)
-    print("\n")
-    print(options)
-    print("\n")
-    print(oh_strategy)
-    print("\n")
-    
-    print("Running global_best...\n")
-
-    optimizer, trace = global_best(steps,                 # step definition
-            rounds=(int(calibration_map['min_rounds']), int(calibration_map['max_rounds'])),                              # min/max number of rounds
-            args=args,                                  # static arguments    
-            n_particles=int(calibration_map['n_particles']),                             # number of particle candidates for each param
-            iters=int(calibration_map['iters']),                                   # max # of iterations
-            n_threads=int(calibration_map['n_threads']),                               # number of threads to use
-            # ftol=0.00000001,                          # min cost function delta for convergence
-            options=options,       # hyperparameter
-            oh_strategy=oh_strategy,   # adaptive hyperparameter adjustments based on current and max # of iterations 
-            conf={
-                'service_timeout': int(calibration_map['service_timeout']),
-                'http_retry': int(calibration_map['http_retry']),
-                'http_allow_redirects': True if calibration_map['allow_redirects'] == "True" else False,
-                'async_call': True if calibration_map['async_call'] == "True" else False,
-                'http_conn_timeout': int(calibration_map['conn_timeout']),
-                'http_read_timeout': int(calibration_map['read_timeout']),
-                'particles_fail': int(calibration_map['particles_fail'])
-                },
-        )  
-    
-    process_queue.put((optimizer, trace))
-
-"""

mgpsogui/util/recosu/pso/csip_access.py

@@ -9,52 +9,34 @@ def csip_worker(reqq: queue.Queue, thread_no: int, stop, full_trace,
     async_call = conf.get('async_call', True)  # default is async
     save_resp = conf.get('save_response_to', None)  # save response, set it to a folder if responses should be saved.
 
-    print("client1")
-
     while not stop():
-        print("client2")
         
         try:
             (rnd, step, iteration, particle, x, step_param_names, calib_params, objfunc, resq) = reqq.get(True, 0.5)
             # print(thread_no, particle)
-
-            print("client3")
     
             c = Client(metainfo=metainfo)
 
-            print("client4")
-
             # static params (from args)
             for param in arg_params:
                 c.add_data(param['name'], param['value'])
 
-            print("client5")
-
             # particle params  (generated from steps)
             # for i, value in enumerate(x):
             for idx, value in enumerate(x[particle, :]):
                 c.add_data(step_param_names[idx], value)
 
-
-            print("client6")
-
             # other, previously calibrated params (other steps)
             for name, value in calib_params.items():
                 c.add_data(name, value)
 
-            print("client7")
-
             # objective function info
             for of in objfunc:
                 c.add_cosu(of['name'], of['of'], of['data'])
                 # c.add_data(of['name'], (of['data'][0], of['data'][1]))
 
-            print("client8")
-
             print('.', end='', flush=True)
 
-            print("client9")
-
             try:
                 # print(c)
                 if async_call:
@@ -62,47 +44,32 @@ def csip_worker(reqq: queue.Queue, thread_no: int, stop, full_trace,
                 else:
                     res = c.execute(url, files=files, conf=conf)
 
-                print("client10")
-
                 if res.is_failed():
                     print(res)
 
-                print("client11")
-
                 if save_resp:
                     res.save_to(os.path.join(save_resp, 'r{}s{}i{}p{}.json'.format(rnd, step, iteration, particle)))
-
-                print("client12")
-
+                    
                 # print(res)
-                print(u'\u2714', end='', flush=True)
-                
-                print("client13")
+                print('O', end='', flush=True)
                 
                 cost = utils.calc_cost(res, objfunc)
 
-                print("client14")
-
                 if full_trace is not None:
                     all_params = {}
                     # for i, value in enumerate(x):
                     for idx, value in enumerate(x[particle, :]):
                         all_params[step_param_names[idx]] = value
-                    
-                    print("client15")
                         
                     for name, value in calib_params.items():
                         all_params[name] = value
                     full_trace.append((all_params, cost))
 
-                print("client16")
-
                 resq.put((particle, cost))
             except Exception as e:
                 print(res)
                 print(e)
-                
-            print("client17")
+
             reqq.task_done()
         except queue.Empty:
             continue

mgpsogui/util/recosu/pso/pso.py

@@ -29,44 +29,30 @@ def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_qu
               step):
     particles = len(x[:, 0])
 
-    print("c1")
-
     pfail_count = conf.get('particles_fail', 1)  # Number of particles allowed to fail.
     pfail_retry = conf.get('particles_retry', 3)  # retry number of times if more than allowed fail
 
-    print("c2")
-
     while pfail_retry > 0:
         cost = np.ones(particles)
         res_queue = queue.Queue()
 
         print('   ', end='', flush=True)
 
-        print("c3")
-
         # submit for processing
         # for i_particle, v in enumerate(x[:, 0]):
         for particle in range(particles):
             req_queue.put((rnd, step, iteration, particle, x, step_param_names, calib_params, step_objfunc, res_queue))
             # req_queue.put((i_particle, x[i_particle,:], step_param_names, calib_params, step_objfunc, res_queue))
 
-        print("c4")
-
         # wait for the cost value to come back
         # for i, v in enumerate(x[:, 0]):
         for idx in range(particles):
-            print("c4.1")
             (particle, p_cost) = res_queue.get()
             cost[particle] = p_cost
-            print("c4.2")
             res_queue.task_done()
 
-        print("c5")
-
         res_queue.join()
 
-        print("c6")
-
         # replace the 'nan' cost values (failed/missing runs) with the mean of the
         # rest of the cost values, hence ignore it
 
@@ -74,24 +60,15 @@ def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_qu
         nan_idx = np.where(np.isnan(cost))
         failed_particles = len(nan_idx[0])
 
-        print("c7")
-
         # leave the loop if fails acceptable
         if failed_particles <= pfail_count:
             break
         
-        print("c8")
-        
         print("Re-running particles, since ", failed_particles, ' out of ', particles, ' particles failed.')
         pfail_retry -= 1
 
-        print("c9")
-
-    print("c10")
-
     if pfail_retry == 0:
         print('Particle evaluation failed ', conf.get('particles_retry', 3), ' times. PSO stopped.')
-        print("c11")
         return None
 
     # print("mean ", mean)
@@ -102,18 +79,6 @@ def eval_cost(x, iteration, step_param_names, step_objfunc, calib_params, req_qu
     print(flush=True)
     return cost
 
-# Simulator
-def global_best_sim(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters: int, options: Dict,
-                oh_strategy: Dict = None, n_threads: int = 4, rtol: float = 0.001, ftol: float = -np.inf,
-                ftol_iter: int = 1, full_trace: List = None, rtol_iter: int = 1,
-                conf: Dict = None, metainfo: Dict = None, cost_target: float = -np.inf, result_queue: Queue = None) -> Tuple:
-    import time
-    while True:
-        print("WOW", flush=True)
-        if result_queue is not None:
-            result_queue.put('WOW')
-        time.sleep(1)
-
 def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters: int, options: Dict,
                 oh_strategy: Dict = None, n_threads: int = 4, rtol: float = 0.001, ftol: float = -np.inf,
                 ftol_iter: int = 1, full_trace: List = None, rtol_iter: int = 1,
@@ -275,16 +240,12 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
             
             args['rnd'] = r + 1
             args['step'] = s + 1
-            
-            print("g1")
 
             # perform optimization
             cost, pos = optimizer[s].optimize(eval_cost, iters=step.get('iters', iters), **args)
             if cost is None:
                 early_exit = True
                 break
-            
-            print("g2")
 
             # capture the best cost
             # if cost < best_cost[s] and np.abs(cost - best_cost[s]) > rtol:
@@ -293,8 +254,6 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
                 no_improvement[s] = False
                 utils.annotate_step(best_cost[s], pos, steps, s)
 
-            print("g3")
-
             print('\n     Step summary, best particle values: {} '.format(pos))
             
             if result_queue is not None:
@@ -306,24 +265,17 @@ def global_best(steps: Dict, rounds: Tuple, args: Dict, n_particles: int, iters:
             step_trace[key]['best_costs'] = best_cost
             step_trace[key]['steps'] = copy.deepcopy(steps)
 
-            print("g4")
-
             if step_file is not None:
                 with open(step_file, "w") as fo:
                     json.dump(step_trace, fo)
 
-            print("g5")
-
             # print(json.dumps(steps, sort_keys=False, indent=2))
 
         if early_exit:
-            print("g6")
             step_trace['exit'] = '1'
             break
 
         round_cost = np.sum(best_cost)
-        
-        print("g7")
 
         # if no improvement in all steps, break out of rounds prematurely
         # but start checking only after min_rounds