ivoryos 0.1.5__py3-none-any.whl

ivoryos/utils/script_runner.py

@@ -0,0 +1,158 @@
+import os
+import csv
+import threading
+import time
+from datetime import datetime
+
+from ivoryos.utils import utils
+from ivoryos.utils.db_models import Script
+from ivoryos.utils.global_config import GlobalConfig
+
+global_config = GlobalConfig()
+global deck
+deck = None
+
+class ScriptRunner:
+    def __init__(self, globals_dict=None):
+        if globals_dict is None:
+            globals_dict = globals()
+        self.globals_dict = globals_dict
+
+        self.stop_event = threading.Event()
+        self.is_running = False
+        self.lock = threading.Lock()
+
+    def reset_stop_event(self):
+        self.stop_event.clear()
+
+    def stop_execution(self):
+        self.stop_event.set()
+        # print("Stop pending tasks")
+
+    def run_script(self, script, repeat_count=1, run_name=None, logger=None, socketio=None, config=None, bo_args=None,
+                   output_path=""):
+        global deck
+        if deck is None:
+            deck = global_config.deck
+        exec_string = script.compile()
+        exec(exec_string)
+        time.sleep(1)
+        with self.lock:
+            if self.is_running:
+                logger.info("System is busy. Please wait for it to finish or stop it before starting a new one.")
+                return None
+            self.is_running = True
+
+        self.reset_stop_event()
+
+        thread = threading.Thread(target=self._run_with_stop_check,
+                                  args=(script, repeat_count, run_name, logger, socketio, config, bo_args, output_path))
+        thread.start()
+        return thread
+
+    def _run_with_stop_check(self, script: Script, repeat_count, run_name, logger, socketio, config, bo_args,
+                             output_path):
+        time.sleep(1)
+        try:
+            # Run "prep" section once
+            script_dict = script.script_dict
+            self._run_actions(script_dict.get("prep", []), section_name="prep", run_name=run_name, logger=logger)
+            output_list = []
+            _, arg_type = script.config("script")
+            _, return_list = script.config_return()
+            # Run "script" section multiple times
+            if repeat_count:
+                self._run_repeat_section(repeat_count, bo_args, output_list, return_list, run_name, logger, socketio)
+            elif config:
+                self._run_config_section(config, arg_type, output_list, return_list, run_name, logger, socketio)
+            # Run "cleanup" section once
+            self._run_actions(script_dict.get("cleanup", []), section_name="cleanup", run_name=run_name, logger=logger)
+            # Save results if necessary
+            if output_list:
+                self._save_results(run_name, arg_type, return_list, output_list, logger, output_path)
+        except Exception as e:
+            logger.error(f"Error during script execution: {e}")
+        finally:
+            with self.lock:
+                self.is_running = False  # Reset the running flag when done
+
+    def _run_actions(self, actions, section_name="", run_name=None, logger=None):
+        logger.info(f'Executing {section_name} steps') if actions else logger.info(f'No {section_name} steps')
+        for action in actions:
+            if self.stop_event.is_set():
+                logger.info(f"Stopping execution during {section_name} section.")
+                break
+            logger.info(f'Executing {action.get("action", "")} action')
+            fname = f"{run_name}_{section_name}"
+            function = self.globals_dict[fname]
+            function()
+
+    def _run_config_section(self, config, arg_type, output_list, return_list, run_name, logger, socketio):
+        compiled = True
+        for i in config:
+            try:
+                i = utils.convert_config_type(i, arg_type)
+            except Exception as e:
+                logger.info(e)
+                compiled = False
+                break
+        if compiled:
+            for i, kwargs in enumerate(config):
+                kwargs = dict(kwargs)
+                if self.stop_event.is_set():
+                    logger.info(f'Stopping execution during {run_name}: {i + 1}/{len(config)}')
+                    break
+                logger.info(f'Executing {i + 1} of {len(config)} with kwargs = {kwargs}')
+                progress = (i + 1) * 100 / len(config)
+                socketio.emit('progress', {'progress': progress})
+                fname = f"{run_name}_script"
+                function = self.globals_dict[fname]
+                output = function(**kwargs)
+                if output:
+                    kwargs.update(output)
+                    output_list.append(kwargs)
+
+    def _run_repeat_section(self, repeat_count, bo_args, output_list, return_list, run_name, logger, socketio):
+        if bo_args:
+            logger.info('Initializing optimizer...')
+            ax_client = utils.ax_initiation(bo_args)
+        for i in range(int(repeat_count)):
+            if self.stop_event.is_set():
+                logger.info(f'Stopping execution during {run_name}: {i + 1}/{int(repeat_count)}')
+                break
+            logger.info(f'Executing {run_name} experiment: {i + 1}/{int(repeat_count)}')
+            progress = (i + 1) * 100 / int(repeat_count)
+            socketio.emit('progress', {'progress': progress})
+            if bo_args:
+                try:
+                    parameters, trial_index = ax_client.get_next_trial()
+                    logger.info(f'Output value: {parameters}')
+                    fname = f"{run_name}_script"
+                    function = self.globals_dict[fname]
+                    output = function(**parameters)
+                    # output = eval(f"{run_name}_script(**{parameters})")
+                    ax_client.complete_trial(trial_index=trial_index, raw_data=output)
+                except Exception as e:
+                    logger.info(f'Optimization error: {e}')
+                    break
+            else:
+                fname = f"{run_name}_script"
+                function = self.globals_dict[fname]
+                output = function()
+
+            if output:
+                output_list.append(output)
+                logger.info(f'Output value: {output}')
+        return output_list
+
+    @staticmethod
+    def _save_results(run_name, arg_type, return_list, output_list, logger, output_path):
+        args = list(arg_type.keys()) if arg_type else []
+        args.extend(return_list)
+        filename = run_name + "_" + datetime.now().strftime("%Y-%m-%d %H-%M") + ".csv"
+        file_path = os.path.join(output_path, filename)
+        with open(file_path, "w", newline='') as file:
+            writer = csv.DictWriter(file, fieldnames=args)
+            writer.writeheader()
+            writer.writerows(output_list)
+        logger.info(f'Results saved to {file_path}')

ivoryos/utils/task_manager.py

@@ -0,0 +1,80 @@
+import threading
+import queue
+import time
+
+
+# A task manager class to manage the queue and tasks
+class TaskManager:
+    def __init__(self):
+        self.task_queue = queue.Queue()
+        self.current_task = None
+        self.stop_event = threading.Event()
+
+    def add_task(self, func, **kwargs):
+        # Add the function and its kwargs to the task queue
+        self.task_queue.put((func, kwargs))
+
+    def run_tasks(self):
+        # Run the tasks from the queue
+        while not self.task_queue.empty():
+            func, kwargs = self.task_queue.get()
+            thread = threading.Thread(target=self.run_task, args=(func, kwargs))
+            self.current_task = thread
+            thread.start()
+            thread.join()  # Wait for the task to finish
+
+    def run_task(self, func, kwargs):
+        # Run the task function with control to stop in the middle
+        self.stop_event.clear()  # Reset the stop flag
+        func(**kwargs)
+        if self.stop_event.is_set():
+            print("Current task was stopped.")
+
+    def stop_current_task(self):
+        # Stop the current task by setting the stop flag
+        if self.current_task and self.current_task.is_alive():
+            print("Stopping current task...")
+            self.stop_event.set()  # Signal to stop the current task
+            self.current_task.join()  # Wait for the task to stop
+
+
+# Wrapping tasks to allow stopping between them
+def function_to_call(stop_event, **kwargs):
+    if stop_event.is_set():
+        return
+    task1(kwargs['arg1'])
+    if stop_event.is_set():
+        return
+    task2(kwargs['arg2'])
+
+
+# Dummy task functions as provided
+def task1(arg1):
+    for i in range(arg1):
+        print(f"Task 1 running: {i}")
+        time.sleep(1)
+
+
+def task2(arg2):
+    for i in range(arg2):
+        print(f"Task 2 running: {i}")
+        time.sleep(1)
+
+
+if __name__ == "__main__":
+    manager = TaskManager()
+
+    # Add tasks to the manager
+    manager.add_task(function_to_call, stop_event=manager.stop_event, arg1=3, arg2=5)
+    manager.add_task(function_to_call, stop_event=manager.stop_event, arg1=2, arg2=4)
+
+    # Run tasks in a separate thread
+    manager_thread = threading.Thread(target=manager.run_tasks)
+    manager_thread.start()
+
+    # Example: Stop the current workflow while task1 is running
+    time.sleep(2)  # Let task1 run for a bit
+    manager.stop_current_task()
+
+    # Wait for all tasks to finish
+    manager_thread.join()

ivoryos/utils/utils.py

@@ -0,0 +1,337 @@
+import importlib
+import inspect
+import logging
+import os
+import pickle
+import subprocess
+import sys
+from typing import Optional, Dict, Tuple
+
+from flask import session
+from flask_socketio import SocketIO
+
+from ivoryos.utils.db_models import Script
+
+
+def get_script_file():
+    session_script = session.get("scripts")
+    if session_script:
+        s = Script()
+        s.__dict__.update(**session_script)
+        return s
+    else:
+        return Script(author=session.get('user'))
+
+
+def post_script_file(script, is_dict=False):
+    if is_dict:
+        session['scripts'] = script
+    else:
+        session['scripts'] = script.as_dict()
+
+
+def create_gui_dir(parent_path):
+    os.makedirs(parent_path, exist_ok=True)
+    for path in ["config_csv", "scripts", "results", "pseudo_deck"]:
+        os.makedirs(os.path.join(parent_path, path), exist_ok=True)
+
+
+def save_to_history(filepath, history_path):
+    connections = []
+    try:
+        with open(history_path, 'r') as file:
+            lines = file.read()
+            connections = lines.split('\n')
+    except FileNotFoundError:
+        pass
+    if filepath not in connections:
+        with open(history_path, 'a') as file:
+            file.writelines(f"{filepath}\n")
+
+
+def import_history(history_path):
+    connections = []
+    try:
+        with open(history_path, 'r') as file:
+            lines = file.read()
+            connections = lines.split('\n')
+    except FileNotFoundError:
+        pass
+    connections = [i for i in connections if not i == '']
+    return connections
+
+
+def available_pseudo_deck(path):
+    import os
+    return os.listdir(path)
+
+
+def parse_functions(class_object=None, debug=False):
+    functions = {}
+    under_score = "_"
+    if debug:
+        under_score = "__"
+    for function, method in inspect.getmembers(type(class_object), predicate=inspect.isfunction):
+        if not function.startswith(under_score) and not function.isupper():
+            try:
+                annotation = inspect.signature(method)
+                # if doc_string:
+                docstring = inspect.getdoc(method)
+                functions[function] = dict(signature=annotation, docstring=docstring)
+
+                # handle getter setters
+                # if callable(att):
+                #     functions[function] = inspect.signature(att)
+                # else:
+                #     att = getattr(class_object.__class__, function)
+                #     if isinstance(att, property) and att.fset is not None:
+                #         setter = att.fset.__annotations__
+                #         setter.pop('return', None)
+                #         if setter:
+                #             functions[function] = setter
+            except Exception:
+                pass
+    return functions
+
+
+def _get_type_from_parameters(arg, parameters):
+    arg_type = ''
+    if type(parameters) is inspect.Signature:
+        p = parameters.parameters
+        # print(p[arg].annotation)
+        if p[arg].annotation is not inspect._empty:
+            # print(p[arg].annotation)
+            if p[arg].annotation.__module__ == 'typing':
+                # print(p[arg].annotation.__args__)
+                arg_type = [i.__name__ for i in p[arg].annotation.__args__]
+            else:
+                arg_type = p[arg].annotation.__name__
+            # print(arg_type)
+    elif type(parameters) is dict:
+        if parameters[arg]:
+
+            if parameters[arg].__module__ == 'typing':
+                arg_type = [i.__name__ for i in parameters[arg].__args__]
+            else:
+                arg_type = parameters[arg].__name__
+    return arg_type
+
+
+def find_variable_in_script(script: Script, args: Dict[str, str]) -> Optional[Tuple[Dict[str, str], Dict[str, str]]]:
+    # TODO: need to search for if the variable exists
+    added_variables: list[Dict[str, str]] = [action for action in script.currently_editing_script if
+                                             action["instrument"] == "variable"]
+
+    possible_variable_arguments = {}
+    possible_variable_types = {}
+
+    for arg_name, arg_val in args.items():
+        for added_variable in added_variables:
+            if added_variable["action"] == arg_val:
+                possible_variable_arguments[arg_name] = added_variable["action"]
+                possible_variable_types[arg_name] = "variable"
+
+    return possible_variable_arguments, possible_variable_types
+
+
+def _convert_by_str(args, arg_types):
+    # print(arg_types)
+    if type(arg_types) is not list:
+        arg_types = [arg_types]
+    for i in arg_types:
+        if i == "any":
+            try:
+                args = eval(args)
+            except Exception:
+                pass
+            return args
+        try:
+            args = eval(f'{i}("{args}")')
+            return args
+        except Exception:
+            raise TypeError(f"Input type error: cannot convert '{args}' to {i}.")
+
+
+def _convert_by_class(args, arg_types):
+    if arg_types.__module__ == 'builtins':
+        args = arg_types(args)
+        return args
+    elif arg_types.__module__ == "typing":
+        for i in arg_types.__args__:  # for typing.Union
+            try:
+                args = i(args)
+                return args
+            except Exception:
+                pass
+        raise TypeError("Input type error.")
+    # else:
+    #     args = globals()[args]
+    return args
+
+
+def convert_config_type(args, arg_types, is_class: bool = False):
+    bool_dict = {"True": True, "False": False}
+    # print(args, arg_types)
+    # print(globals())
+    if args:
+        for arg in args:
+            if arg not in arg_types.keys():
+                raise ValueError("config file format not supported.")
+            if args[arg] == '' or args[arg] == "None":
+                args[arg] = None
+            elif args[arg] == "True" or args[arg] == "False":
+                args[arg] = bool_dict[args[arg]]
+            else:
+                arg_type = arg_types[arg]
+
+                if is_class:
+                    # if arg_type.__module__ == 'builtins':
+                    args[arg] = _convert_by_class(args[arg], arg_type)
+                else:
+                    args[arg] = _convert_by_str(args[arg], arg_type)
+    return args
+
+
+def import_module_by_filepath(filepath: str, name: str):
+    spec = importlib.util.spec_from_file_location(name, filepath)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class SocketIOHandler(logging.Handler):
+    def __init__(self, socketio: SocketIO):
+        super().__init__()
+        self.formatter = logging.Formatter('%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+        self.socketio = socketio
+
+    def emit(self, record):
+        message = self.format(record)
+        # session["last_log"] = message
+        self.socketio.emit('log', {'message': message})
+
+
+def start_logger(socketio: SocketIO, logger_name: str, log_filename: str = None):
+    # logging.basicConfig( format='%(asctime)s - %(message)s')
+    formatter = logging.Formatter(fmt='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(logging.INFO)
+    file_handler = logging.FileHandler(filename=log_filename, )
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+    # console_logger = logging.StreamHandler()  # stream to console
+    # logger.addHandler(console_logger)
+    socketio_handler = SocketIOHandler(socketio)
+    logger.addHandler(socketio_handler)
+    return logger
+
+
+def ax_wrapper(data):
+    from ax.service.utils.instantiation import ObjectiveProperties
+    parameter = []
+    objectives = {}
+    # Iterate through the webui_data dictionary
+    for key, value in data.items():
+        # Check if the key corresponds to a parameter type
+        if "_type" in key:
+            param_name = key.split("_type")[0]
+            param_type = value
+            param_value = data[f"{param_name}_value"].split(",")
+            try:
+                values = [float(v) for v in param_value]
+            except Exception:
+                values = param_value
+            if param_type == "range":
+                parameter.append({"name": param_name, "type": param_type, "bounds": values})
+            if param_type == "choice":
+                parameter.append({"name": param_name, "type": param_type, "values": values})
+            if param_type == "fixed":
+                parameter.append({"name": param_name, "type": param_type, "value": values[0]})
+        elif key.endswith("_min"):
+            if not value == 'none':
+                obj_name = key.split("_min")[0]
+                is_min = True if value == "minimize" else False
+
+                threshold = None if f"{obj_name}_threshold" not in data else data[f"{obj_name}_threshold"]
+                properties = ObjectiveProperties(minimize=is_min, threshold=threshold)
+                objectives[obj_name] = properties
+    return parameter, objectives
+
+
+def ax_initiation(data):
+    install_and_import("ax", "ax-platform")
+    parameter, objectives = ax_wrapper(data)
+    from ax.service.ax_client import AxClient
+    ax_client = AxClient()
+    ax_client.create_experiment(parameter, objectives)
+    return ax_client
+
+
+def get_arg_type(args, parameters):
+    arg_types = {}
+    # print(args, parameters)
+    if args:
+        for arg in args:
+            arg_types[arg] = _get_type_from_parameters(arg, parameters)
+    return arg_types
+
+
+def install_and_import(package, package_name=None):
+    try:
+        # Check if the package is already installed
+        importlib.import_module(package)
+        # print(f"{package} is already installed.")
+    except ImportError:
+        # If not installed, install it
+        # print(f"{package} is not installed. Installing now...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", package_name or package])
+        # print(f"{package} has been installed successfully.")
+
+
+def process_data(data, config_type):
+    rows = {}  # Dictionary to hold webui_data organized by rows
+
+    # Organize webui_data by rows
+    for key, value in data.items():
+        if value:  # Only process non-empty values
+            # Extract the field name and row index
+            field_name, row_index = key.split('[')
+            row_index = int(row_index.rstrip(']'))
+
+            # If row not in rows, create a new dictionary for that row
+            if row_index not in rows:
+                rows[row_index] = {}
+
+            # Add or update the field value in the specific row's dictionary
+            rows[row_index][field_name] = value
+
+    # Filter out any empty rows and create a list of dictionaries
+    filtered_rows = [row for row in rows.values() if len(row) == len(config_type)]
+
+    return filtered_rows
+
+
+def parse_deck(deck, save=False, output_path=''):
+    deck_variables = {f"deck.{name}": parse_functions(val) for name, val in vars(deck).items()
+                      if not type(val).__module__ == 'builtins'
+                      and not name[0].isupper()
+                      and not name.startswith("_")}
+    if deck_variables and save:
+        # pseudo_deck = parse_dict
+        parse_dict = deck_variables.copy()
+        parse_dict["deck_name"] = os.path.splitext(os.path.basename(deck.__file__))[
+            0] if deck.__name__ == "__main__" else deck.__name__
+        with open(os.path.join(output_path, f"{parse_dict['deck_name']}.pkl"), 'wb') as file:
+            pickle.dump(parse_dict, file)
+    return deck_variables
+
+
+def load_deck(pkl_name):
+    if not pkl_name:
+        return None
+    try:
+        with open(pkl_name, 'rb') as f:
+            pseudo_deck = pickle.load(f)
+        return pseudo_deck
+    except FileNotFoundError:
+        return None

ivoryos-0.1.5.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2024 Ivory Zhang | Hein Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

ivoryos-0.1.5.dist-info/METADATA

@@ -0,0 +1,96 @@
+Metadata-Version: 2.1
+Name: ivoryos
+Version: 0.1.5
+Summary: an open-source Python package enabling Self-Driving Labs (SDLs) interoperability
+Home-page: https://gitlab.com/heingroup/ivoryos
+Author: Ivory Zhang
+Author-email: ivoryzhang@chem.ubc.ca
+License: MIT
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bcrypt
+Requires-Dist: Flask-Login
+Requires-Dist: Flask-Session
+Requires-Dist: Flask-SocketIO
+Requires-Dist: Flask-SQLAlchemy
+Requires-Dist: Flask-WTF
+Requires-Dist: SQLAlchemy-Utils
+Requires-Dist: python-dotenv
+
+![](https://gitlab.com/heingroup/ivoryos/raw/main/docs/ivoryos.png)
+# ivoryOS: interoperable Web UI for self-driving laboratories (SDLs)
+ivoryOS is a "plug and play" web UI extension for flexible SDLs, enabling interoperability between SDLs.
+## Description
+Granting SDLs flexibility and modularity makes it almost impossible to design a UI, yet it's a necessity for allowing more people to interact with it (democratisation). 
+This web UI aims to ease up the control of any Python-based SDLs by displaying functions and parameters for initialized modules dynamically. 
+The modules can be hardware API, high-level functions, or experiment workflow.
+With the least modification of the current workflow, user can design, manage and execute their experimental designs and monitor the execution process. 
+## AI assistant
+To streamline the experimental design on SDLs, we also integrate Large Language Models (LLMs) to interpret the inspected functions and generate code according to task descriptions.
+
+## Installation
+```
+pip install ivoryos
+```
+
+## Usage
+### Quick start
+In your SDL script, use `ivoryos(__name__)`. Example in [abstract_sdl.py](https://gitlab.com/heingroup/ivoryos/-/blob/main/example/dummy_ur/dummy_deck.py)
+
+```python
+import ivoryos
+
+ivoryos.run(__name__)
+```
+
+### Additional settings
+#### Enable LLMs with [OpenAI API](https://github.com/openai/openai-python)
+1. Create a `.env` file for `OPENAI_API_KEY`
+```
+OPENAI_API_KEY="Your API Key"
+```
+2. In your SDL script, define model, you can use any GPT models.
+
+```python
+ivoryos.run(__name__, model="gpt-3.5-turbo")
+```
+
+#### Enable local LLMs with [Ollama](https://ollama.com/)
+1. Download Ollama.
+2. pull models from Ollama
+3. In your SDL script, define LLM server and model, you can use any models available on Ollama.
+
+```python
+ivoryos.run(__name__, llm_server="localhost", model="llama3.1")
+```
+
+#### Add additional logger(s)
+```python
+ivoryos.run(__name__, logger="logger name")
+```
+or
+```python
+ivoryos.run(__name__, logger=["logger 1", "logger 2"])
+```
+#### Offline (design without hardware connection)
+After one successful connection, a blueprint will be automatically saved and made accessible without hardware connection. In a new Python script in the same directory, use `ivoryos.run()` to start offline mode.
+
+```python
+ivoryos.run()
+```
+## Deck snapshot example
+![](https://gitlab.com/heingroup/ivoryos/raw/main/docs/demo.gif)
+## Developing
+This is a wip project. Here are some future actions.
+1. Support @setter decorator.
+2. Documentation: white paper wip
+3. Compatibility: compatability report to open-source lab hardware APIs will soon be added. As of now, due to the limitation of web form, the usability of APIs with object inputs (e.g. Opentron Python API) is very limited.
+
+
+## Authors and Acknowledgement
+Ivory Zhang, Lucy Hao
+
+Authors acknowledge all former and current Hein Lab members for their valuable suggestions. 
+
+## License
+[LICENSE](LICENSE)