ivoryos 1.0.9__py3-none-any.whl → 1.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/source/conf.py +84 -0
- ivoryos/__init__.py +17 -207
- ivoryos/app.py +154 -0
- ivoryos/config.py +1 -0
- ivoryos/optimizer/ax_optimizer.py +191 -0
- ivoryos/optimizer/base_optimizer.py +84 -0
- ivoryos/optimizer/baybe_optimizer.py +193 -0
- ivoryos/optimizer/nimo_optimizer.py +173 -0
- ivoryos/optimizer/registry.py +11 -0
- ivoryos/routes/auth/auth.py +43 -14
- ivoryos/routes/auth/templates/change_password.html +32 -0
- ivoryos/routes/control/control.py +101 -366
- ivoryos/routes/control/control_file.py +33 -0
- ivoryos/routes/control/control_new_device.py +152 -0
- ivoryos/routes/control/templates/controllers.html +193 -0
- ivoryos/routes/control/templates/controllers_new.html +112 -0
- ivoryos/routes/control/utils.py +40 -0
- ivoryos/routes/data/data.py +197 -0
- ivoryos/routes/data/templates/components/step_card.html +78 -0
- ivoryos/routes/{database/templates/database → data/templates}/workflow_database.html +14 -8
- ivoryos/routes/data/templates/workflow_view.html +360 -0
- ivoryos/routes/design/__init__.py +4 -0
- ivoryos/routes/design/design.py +348 -657
- ivoryos/routes/design/design_file.py +68 -0
- ivoryos/routes/design/design_step.py +171 -0
- ivoryos/routes/design/templates/components/action_form.html +53 -0
- ivoryos/routes/design/templates/components/actions_panel.html +25 -0
- ivoryos/routes/design/templates/components/autofill_toggle.html +10 -0
- ivoryos/routes/design/templates/components/canvas.html +5 -0
- ivoryos/routes/design/templates/components/canvas_footer.html +9 -0
- ivoryos/routes/design/templates/components/canvas_header.html +75 -0
- ivoryos/routes/design/templates/components/canvas_main.html +39 -0
- ivoryos/routes/design/templates/components/deck_selector.html +10 -0
- ivoryos/routes/design/templates/components/edit_action_form.html +53 -0
- ivoryos/routes/design/templates/components/info_modal.html +318 -0
- ivoryos/routes/design/templates/components/instruments_panel.html +88 -0
- ivoryos/routes/design/templates/components/modals/drop_modal.html +17 -0
- ivoryos/routes/design/templates/components/modals/json_modal.html +22 -0
- ivoryos/routes/design/templates/components/modals/new_script_modal.html +17 -0
- ivoryos/routes/design/templates/components/modals/rename_modal.html +23 -0
- ivoryos/routes/design/templates/components/modals/saveas_modal.html +27 -0
- ivoryos/routes/design/templates/components/modals.html +6 -0
- ivoryos/routes/design/templates/components/python_code_overlay.html +56 -0
- ivoryos/routes/design/templates/components/sidebar.html +15 -0
- ivoryos/routes/design/templates/components/text_to_code_panel.html +20 -0
- ivoryos/routes/design/templates/experiment_builder.html +44 -0
- ivoryos/routes/execute/__init__.py +0 -0
- ivoryos/routes/execute/execute.py +377 -0
- ivoryos/routes/execute/execute_file.py +78 -0
- ivoryos/routes/execute/templates/components/error_modal.html +20 -0
- ivoryos/routes/execute/templates/components/logging_panel.html +56 -0
- ivoryos/routes/execute/templates/components/progress_panel.html +27 -0
- ivoryos/routes/execute/templates/components/run_panel.html +9 -0
- ivoryos/routes/execute/templates/components/run_tabs.html +60 -0
- ivoryos/routes/execute/templates/components/tab_bayesian.html +520 -0
- ivoryos/routes/execute/templates/components/tab_configuration.html +383 -0
- ivoryos/routes/execute/templates/components/tab_repeat.html +18 -0
- ivoryos/routes/execute/templates/experiment_run.html +30 -0
- ivoryos/routes/library/__init__.py +0 -0
- ivoryos/routes/library/library.py +157 -0
- ivoryos/routes/{database/templates/database/scripts_database.html → library/templates/library.html} +32 -23
- ivoryos/routes/main/main.py +31 -3
- ivoryos/routes/main/templates/{main/home.html → home.html} +4 -4
- ivoryos/server.py +180 -0
- ivoryos/socket_handlers.py +52 -0
- ivoryos/static/ivoryos_logo.png +0 -0
- ivoryos/static/js/action_handlers.js +384 -0
- ivoryos/static/js/db_delete.js +23 -0
- ivoryos/static/js/script_metadata.js +39 -0
- ivoryos/static/js/socket_handler.js +40 -5
- ivoryos/static/js/sortable_design.js +107 -56
- ivoryos/static/js/ui_state.js +114 -0
- ivoryos/templates/base.html +67 -8
- ivoryos/utils/bo_campaign.py +180 -3
- ivoryos/utils/client_proxy.py +267 -36
- ivoryos/utils/db_models.py +300 -65
- ivoryos/utils/decorators.py +34 -0
- ivoryos/utils/form.py +63 -29
- ivoryos/utils/global_config.py +34 -1
- ivoryos/utils/nest_script.py +314 -0
- ivoryos/utils/py_to_json.py +295 -0
- ivoryos/utils/script_runner.py +599 -165
- ivoryos/utils/serilize.py +201 -0
- ivoryos/utils/task_runner.py +71 -21
- ivoryos/utils/utils.py +50 -6
- ivoryos/version.py +1 -1
- ivoryos-1.4.4.dist-info/METADATA +263 -0
- ivoryos-1.4.4.dist-info/RECORD +119 -0
- {ivoryos-1.0.9.dist-info → ivoryos-1.4.4.dist-info}/WHEEL +1 -1
- {ivoryos-1.0.9.dist-info → ivoryos-1.4.4.dist-info}/top_level.txt +1 -0
- tests/unit/test_type_conversion.py +42 -0
- tests/unit/test_util.py +3 -0
- ivoryos/routes/control/templates/control/controllers.html +0 -78
- ivoryos/routes/control/templates/control/controllers_home.html +0 -55
- ivoryos/routes/control/templates/control/controllers_new.html +0 -89
- ivoryos/routes/database/database.py +0 -306
- ivoryos/routes/database/templates/database/step_card.html +0 -7
- ivoryos/routes/database/templates/database/workflow_view.html +0 -130
- ivoryos/routes/design/templates/design/experiment_builder.html +0 -521
- ivoryos/routes/design/templates/design/experiment_run.html +0 -558
- ivoryos-1.0.9.dist-info/METADATA +0 -218
- ivoryos-1.0.9.dist-info/RECORD +0 -61
- /ivoryos/routes/auth/templates/{auth/login.html → login.html} +0 -0
- /ivoryos/routes/auth/templates/{auth/signup.html → signup.html} +0 -0
- /ivoryos/routes/{database → data}/__init__.py +0 -0
- /ivoryos/routes/main/templates/{main/help.html → help.html} +0 -0
- {ivoryos-1.0.9.dist-info → ivoryos-1.4.4.dist-info/licenses}/LICENSE +0 -0
docs/source/conf.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Configuration file for the Sphinx documentation builder.
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import urllib
|
|
5
|
+
|
|
6
|
+
import requests
|
|
7
|
+
|
|
8
|
+
# -- General configuration
|
|
9
|
+
sys.path.insert(0, os.path.abspath('../../'))
|
|
10
|
+
from ivoryos.version import __version__
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# appending suite readme.rst to doc
|
|
14
|
+
|
|
15
|
+
external_readme = [
|
|
16
|
+
{
|
|
17
|
+
"name": 'plugin.rst',
|
|
18
|
+
"url": "https://gitlab.com/heingroup/ivoryos-suite/ivoryos-plugin-template/-/raw/main/README.rst"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"name": 'client.rst',
|
|
22
|
+
"url": "https://gitlab.com/heingroup/ivoryos-suite/ivoryos-client/-/raw/main/README.rst"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"name": 'mcp.rst',
|
|
26
|
+
"url": "https://gitlab.com/heingroup/ivoryos-suite/ivoryos-mcp/-/raw/main/README.rst"
|
|
27
|
+
}
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
for item in external_readme:
|
|
31
|
+
readme_url = item['url']
|
|
32
|
+
name = item['name']
|
|
33
|
+
output_path = os.path.join(os.path.dirname(__file__), name)
|
|
34
|
+
r = requests.get(readme_url, verify=False)
|
|
35
|
+
if not os.path.exists(output_path):
|
|
36
|
+
with open(output_path, "wb") as f:
|
|
37
|
+
f.write(r.content)
|
|
38
|
+
|
|
39
|
+
# -- Project information
|
|
40
|
+
project = 'ivoryOS'
|
|
41
|
+
copyright = '2024, Ivory Zhang'
|
|
42
|
+
author = 'Ivory Zhang, Lucy Hao'
|
|
43
|
+
version = __version__
|
|
44
|
+
|
|
45
|
+
extensions = [
|
|
46
|
+
'sphinx.ext.duration',
|
|
47
|
+
'sphinx.ext.doctest',
|
|
48
|
+
'sphinx.ext.autosummary',
|
|
49
|
+
'sphinx.ext.intersphinx',
|
|
50
|
+
'sphinxcontrib.httpdomain',
|
|
51
|
+
'sphinxcontrib.autohttp.flask',
|
|
52
|
+
'sphinxcontrib.autohttp.flaskqref'
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
install_requires = [
|
|
56
|
+
'sphinx-autodoc-typehints'
|
|
57
|
+
]
|
|
58
|
+
autodoc_mock_imports = ["flask_sqlalchemy", "another_hard_to_import_lib"]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
intersphinx_mapping = {
|
|
62
|
+
'python': ('https://docs.python.org/3/', None),
|
|
63
|
+
'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
|
|
64
|
+
}
|
|
65
|
+
intersphinx_disabled_domains = ['std']
|
|
66
|
+
|
|
67
|
+
templates_path = ['_templates']
|
|
68
|
+
|
|
69
|
+
html_static_path = ['_static']
|
|
70
|
+
html_css_files = [
|
|
71
|
+
'custom.css',
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
html_allow_raw_html = True
|
|
75
|
+
|
|
76
|
+
# -- Options for HTML output
|
|
77
|
+
|
|
78
|
+
html_theme = 'sphinx_rtd_theme'
|
|
79
|
+
|
|
80
|
+
# -- Options for EPUB output
|
|
81
|
+
epub_show_urls = 'footnote'
|
|
82
|
+
|
|
83
|
+
# The master toctree document.
|
|
84
|
+
master_doc = 'index'
|
ivoryos/__init__.py
CHANGED
|
@@ -1,208 +1,18 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
from typing import Union
|
|
4
|
-
|
|
5
|
-
from flask import Flask, redirect, url_for, g, Blueprint
|
|
6
|
-
|
|
7
|
-
from ivoryos.config import Config, get_config
|
|
8
|
-
from ivoryos.routes.auth.auth import auth, login_manager
|
|
9
|
-
from ivoryos.routes.control.control import control
|
|
10
|
-
from ivoryos.routes.database.database import database
|
|
11
|
-
from ivoryos.routes.design.design import design, socketio
|
|
12
|
-
from ivoryos.routes.main.main import main
|
|
13
|
-
# from ivoryos.routes.monitor.monitor import monitor
|
|
14
|
-
from ivoryos.utils import utils
|
|
15
|
-
from ivoryos.utils.db_models import db, User
|
|
16
|
-
from ivoryos.utils.global_config import GlobalConfig
|
|
17
|
-
from ivoryos.utils.script_runner import ScriptRunner
|
|
1
|
+
from ivoryos.server import run, global_config
|
|
2
|
+
from ivoryos.optimizer.registry import OPTIMIZER_REGISTRY
|
|
18
3
|
from ivoryos.version import __version__ as ivoryos_version
|
|
19
|
-
from
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
url_prefix = os.getenv('URL_PREFIX', "/ivoryos")
|
|
36
|
-
app = Flask(__name__, static_url_path=f'{url_prefix}/static', static_folder='static')
|
|
37
|
-
app.register_blueprint(main, url_prefix=url_prefix)
|
|
38
|
-
app.register_blueprint(auth, url_prefix=url_prefix)
|
|
39
|
-
app.register_blueprint(control, url_prefix=url_prefix)
|
|
40
|
-
app.register_blueprint(design, url_prefix=url_prefix)
|
|
41
|
-
app.register_blueprint(database, url_prefix=url_prefix)
|
|
42
|
-
|
|
43
|
-
@login_manager.user_loader
|
|
44
|
-
def load_user(user_id):
|
|
45
|
-
"""
|
|
46
|
-
This function is called by Flask-Login on every request to get the
|
|
47
|
-
current user object from the user ID stored in the session.
|
|
48
|
-
"""
|
|
49
|
-
# The correct implementation is to fetch the user from the database.
|
|
50
|
-
return db.session.get(User, user_id)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def create_app(config_class=None):
|
|
54
|
-
"""
|
|
55
|
-
create app, init database
|
|
56
|
-
"""
|
|
57
|
-
app.config.from_object(config_class or 'config.get_config()')
|
|
58
|
-
os.makedirs(app.config["OUTPUT_FOLDER"], exist_ok=True)
|
|
59
|
-
# Initialize extensions
|
|
60
|
-
socketio.init_app(app, cors_allowed_origins="*", cookie=None)
|
|
61
|
-
login_manager.init_app(app)
|
|
62
|
-
login_manager.login_view = "auth.login"
|
|
63
|
-
db.init_app(app)
|
|
64
|
-
|
|
65
|
-
# Create database tables
|
|
66
|
-
with app.app_context():
|
|
67
|
-
db.create_all()
|
|
68
|
-
|
|
69
|
-
# Additional setup
|
|
70
|
-
utils.create_gui_dir(app.config['OUTPUT_FOLDER'])
|
|
71
|
-
|
|
72
|
-
# logger_list = app.config["LOGGERS"]
|
|
73
|
-
logger_path = os.path.join(app.config["OUTPUT_FOLDER"], app.config["LOGGERS_PATH"])
|
|
74
|
-
logger = utils.start_logger(socketio, 'gui_logger', logger_path)
|
|
75
|
-
|
|
76
|
-
@app.before_request
|
|
77
|
-
def before_request():
|
|
78
|
-
"""
|
|
79
|
-
Called before
|
|
80
|
-
|
|
81
|
-
"""
|
|
82
|
-
g.logger = logger
|
|
83
|
-
g.socketio = socketio
|
|
84
|
-
|
|
85
|
-
@app.route('/')
|
|
86
|
-
def redirect_to_prefix():
|
|
87
|
-
return redirect(url_for('main.index', version=ivoryos_version)) # Assuming 'index' is a route in your blueprint
|
|
88
|
-
|
|
89
|
-
return app
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def run(module=None, host="0.0.0.0", port=None, debug=None, llm_server=None, model=None,
|
|
93
|
-
config: Config = None,
|
|
94
|
-
logger: Union[str, list] = None,
|
|
95
|
-
logger_output_name: str = None,
|
|
96
|
-
enable_design: bool = True,
|
|
97
|
-
blueprint_plugins: Union[list, Blueprint] = [],
|
|
98
|
-
exclude_names: list = [],
|
|
99
|
-
):
|
|
100
|
-
"""
|
|
101
|
-
Start ivoryOS app server.
|
|
102
|
-
|
|
103
|
-
:param module: module name, __name__ for current module
|
|
104
|
-
:param host: host address, defaults to 0.0.0.0
|
|
105
|
-
:param port: port, defaults to None, and will use 8000
|
|
106
|
-
:param debug: debug mode, defaults to None (True)
|
|
107
|
-
:param llm_server: llm server, defaults to None.
|
|
108
|
-
:param model: llm model, defaults to None. If None, app will run without text-to-code feature
|
|
109
|
-
:param config: config class, defaults to None
|
|
110
|
-
:param logger: logger name of list of logger names, defaults to None
|
|
111
|
-
:param logger_output_name: log file save name of logger, defaults to None, and will use "default.log"
|
|
112
|
-
:param enable_design: enable design canvas, database and workflow execution
|
|
113
|
-
:param blueprint_plugins: Union[list[Blueprint], Blueprint] custom Blueprint pages
|
|
114
|
-
:param exclude_names: list[str] module names to exclude from parsing
|
|
115
|
-
"""
|
|
116
|
-
app = create_app(config_class=config or get_config()) # Create app instance using factory function
|
|
117
|
-
|
|
118
|
-
# plugins = load_installed_plugins(app, socketio)
|
|
119
|
-
plugins = []
|
|
120
|
-
if blueprint_plugins:
|
|
121
|
-
config_plugins = load_plugins(blueprint_plugins, app, socketio)
|
|
122
|
-
plugins.extend(config_plugins)
|
|
123
|
-
|
|
124
|
-
def inject_nav_config():
|
|
125
|
-
"""Make NAV_CONFIG available globally to all templates."""
|
|
126
|
-
return dict(
|
|
127
|
-
enable_design=enable_design,
|
|
128
|
-
plugins=plugins,
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
app.context_processor(inject_nav_config)
|
|
132
|
-
port = port or int(os.environ.get("PORT", 8000))
|
|
133
|
-
debug = debug if debug is not None else app.config.get('DEBUG', True)
|
|
134
|
-
|
|
135
|
-
app.config["LOGGERS"] = logger
|
|
136
|
-
app.config["LOGGERS_PATH"] = logger_output_name or app.config["LOGGERS_PATH"] # default.log
|
|
137
|
-
logger_path = os.path.join(app.config["OUTPUT_FOLDER"], app.config["LOGGERS_PATH"])
|
|
138
|
-
dummy_deck_path = os.path.join(app.config["OUTPUT_FOLDER"], app.config["DUMMY_DECK"])
|
|
139
|
-
|
|
140
|
-
if module:
|
|
141
|
-
app.config["MODULE"] = module
|
|
142
|
-
app.config["OFF_LINE"] = False
|
|
143
|
-
global_config.deck = sys.modules[module]
|
|
144
|
-
global_config.deck_snapshot = utils.create_deck_snapshot(global_config.deck,
|
|
145
|
-
output_path=dummy_deck_path,
|
|
146
|
-
save=True,
|
|
147
|
-
exclude_names=exclude_names
|
|
148
|
-
)
|
|
149
|
-
else:
|
|
150
|
-
app.config["OFF_LINE"] = True
|
|
151
|
-
if model:
|
|
152
|
-
app.config["ENABLE_LLM"] = True
|
|
153
|
-
app.config["LLM_MODEL"] = model
|
|
154
|
-
app.config["LLM_SERVER"] = llm_server
|
|
155
|
-
utils.install_and_import('openai')
|
|
156
|
-
from ivoryos.utils.llm_agent import LlmAgent
|
|
157
|
-
global_config.agent = LlmAgent(host=llm_server, model=model,
|
|
158
|
-
output_path=app.config["OUTPUT_FOLDER"] if module is not None else None)
|
|
159
|
-
else:
|
|
160
|
-
app.config["ENABLE_LLM"] = False
|
|
161
|
-
if logger and type(logger) is str:
|
|
162
|
-
utils.start_logger(socketio, log_filename=logger_path, logger_name=logger)
|
|
163
|
-
elif type(logger) is list:
|
|
164
|
-
for log in logger:
|
|
165
|
-
utils.start_logger(socketio, log_filename=logger_path, logger_name=log)
|
|
166
|
-
|
|
167
|
-
# in case Python 3.12 or higher doesn't log URL
|
|
168
|
-
if sys.version_info >= (3, 12):
|
|
169
|
-
ip = utils.get_local_ip()
|
|
170
|
-
print(f"Server running at http://localhost:{port}")
|
|
171
|
-
if not ip == "127.0.0.1":
|
|
172
|
-
print(f"Server running at http://{ip}:{port}")
|
|
173
|
-
socketio.run(app, host=host, port=port, debug=debug, use_reloader=False, allow_unsafe_werkzeug=True)
|
|
174
|
-
# return app
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
def load_installed_plugins(app, socketio):
|
|
178
|
-
"""
|
|
179
|
-
Dynamically load installed plugins and attach Flask-SocketIO.
|
|
180
|
-
"""
|
|
181
|
-
plugin_names = []
|
|
182
|
-
for entry_point in entry_points().get("ivoryos.plugins", []):
|
|
183
|
-
plugin = entry_point.load()
|
|
184
|
-
|
|
185
|
-
# If the plugin has an `init_socketio()` function, pass socketio
|
|
186
|
-
if hasattr(plugin, 'init_socketio'):
|
|
187
|
-
plugin.init_socketio(socketio)
|
|
188
|
-
|
|
189
|
-
plugin_names.append(entry_point.name)
|
|
190
|
-
app.register_blueprint(getattr(plugin, entry_point.name), url_prefix=f"{url_prefix}/{entry_point.name}")
|
|
191
|
-
|
|
192
|
-
return plugin_names
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
def load_plugins(blueprints: Union[list, Blueprint], app, socketio):
|
|
196
|
-
"""
|
|
197
|
-
Dynamically load installed plugins and attach Flask-SocketIO.
|
|
198
|
-
"""
|
|
199
|
-
plugin_names = []
|
|
200
|
-
if not isinstance(blueprints, list):
|
|
201
|
-
blueprints = [blueprints]
|
|
202
|
-
for blueprint in blueprints:
|
|
203
|
-
# If the plugin has an `init_socketio()` function, pass socketio
|
|
204
|
-
if hasattr(blueprint, 'init_socketio'):
|
|
205
|
-
blueprint.init_socketio(socketio)
|
|
206
|
-
plugin_names.append(blueprint.name)
|
|
207
|
-
app.register_blueprint(blueprint, url_prefix=f"{url_prefix}/{blueprint.name}")
|
|
208
|
-
return plugin_names
|
|
4
|
+
from ivoryos.utils.decorators import block, BUILDING_BLOCKS
|
|
5
|
+
from ivoryos.app import app, create_app, socketio, db
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"block",
|
|
9
|
+
"BUILDING_BLOCKS",
|
|
10
|
+
"OPTIMIZER_REGISTRY",
|
|
11
|
+
"run",
|
|
12
|
+
"app",
|
|
13
|
+
"ivoryos_version",
|
|
14
|
+
"create_app",
|
|
15
|
+
"socketio",
|
|
16
|
+
"global_config",
|
|
17
|
+
"db"
|
|
18
|
+
]
|
ivoryos/app.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
import bcrypt
|
|
5
|
+
from flask import Flask, session, g, redirect, url_for
|
|
6
|
+
from flask_login import AnonymousUserMixin
|
|
7
|
+
|
|
8
|
+
from ivoryos.utils import utils
|
|
9
|
+
from ivoryos.utils.db_models import db, User
|
|
10
|
+
from ivoryos.routes.auth.auth import auth, login_manager
|
|
11
|
+
from ivoryos.routes.control.control import control
|
|
12
|
+
from ivoryos.routes.data.data import data
|
|
13
|
+
from ivoryos.routes.library.library import library
|
|
14
|
+
from ivoryos.routes.design.design import design
|
|
15
|
+
from ivoryos.routes.execute.execute import execute
|
|
16
|
+
from ivoryos.socket_handlers import socketio
|
|
17
|
+
from ivoryos.routes.main.main import main
|
|
18
|
+
from ivoryos.version import __version__ as ivoryos_version
|
|
19
|
+
from sqlalchemy import inspect, text
|
|
20
|
+
|
|
21
|
+
url_prefix = os.getenv('URL_PREFIX', "/ivoryos")
|
|
22
|
+
app = Flask(__name__, static_url_path=f'{url_prefix}/static', static_folder='static')
|
|
23
|
+
app.register_blueprint(main, url_prefix=url_prefix)
|
|
24
|
+
app.register_blueprint(auth, url_prefix=f'{url_prefix}/{auth.name}')
|
|
25
|
+
app.register_blueprint(library, url_prefix=f'{url_prefix}/{library.name}')
|
|
26
|
+
app.register_blueprint(control, url_prefix=f'{url_prefix}/instruments')
|
|
27
|
+
app.register_blueprint(design, url_prefix=f'{url_prefix}')
|
|
28
|
+
app.register_blueprint(execute, url_prefix=f'{url_prefix}')
|
|
29
|
+
app.register_blueprint(data, url_prefix=f'{url_prefix}')
|
|
30
|
+
# app.register_blueprint(api, url_prefix=f'{url_prefix}/{api.name}')
|
|
31
|
+
|
|
32
|
+
def reset_old_schema(engine, db_dir):
|
|
33
|
+
inspector = inspect(engine)
|
|
34
|
+
tables = inspector.get_table_names()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# Check if old tables exist (no workflow_phases table)
|
|
38
|
+
has_workflow_phase = 'workflow_phases' in tables
|
|
39
|
+
old_workflow_run = 'workflow_runs' in tables
|
|
40
|
+
old_workflow_step = 'workflow_steps' in tables
|
|
41
|
+
|
|
42
|
+
# v1.3.4 only delete and backup when there is runs but no phases
|
|
43
|
+
if not has_workflow_phase and old_workflow_run:
|
|
44
|
+
print("⚠️ Old workflow database detected! All previous workflows have been reset to support the new schema.")
|
|
45
|
+
# Backup old DB
|
|
46
|
+
db_path = os.path.join(db_dir, "ivoryos.db")
|
|
47
|
+
if os.path.exists(db_path):
|
|
48
|
+
# os.makedirs(backup_dir, exist_ok=True)
|
|
49
|
+
from datetime import datetime
|
|
50
|
+
import shutil
|
|
51
|
+
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
52
|
+
backup_path = os.path.join(db_dir, f"ivoryos_backup_{ts}.db")
|
|
53
|
+
shutil.copy(db_path, backup_path)
|
|
54
|
+
print(f"Backup created at {backup_path}")
|
|
55
|
+
with engine.begin() as conn:
|
|
56
|
+
# Drop old tables
|
|
57
|
+
if old_workflow_step:
|
|
58
|
+
conn.execute(text("DROP TABLE IF EXISTS workflow_steps"))
|
|
59
|
+
if old_workflow_run:
|
|
60
|
+
conn.execute(text("DROP TABLE IF EXISTS workflow_runs"))
|
|
61
|
+
with engine.begin() as conn:
|
|
62
|
+
try:
|
|
63
|
+
conn.execute(
|
|
64
|
+
text("ALTER TABLE user ADD COLUMN settings TEXT")
|
|
65
|
+
)
|
|
66
|
+
except Exception:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
# Recreate new schema
|
|
70
|
+
db.create_all() # creates workflow_runs, workflow_phases, workflow_steps
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def create_admin():
|
|
74
|
+
"""
|
|
75
|
+
Create an admin user with username 'admin' and password 'admin' if it doesn't exist.
|
|
76
|
+
"""
|
|
77
|
+
with app.app_context():
|
|
78
|
+
admin_user = User.query.filter_by(username='admin').first()
|
|
79
|
+
if not admin_user:
|
|
80
|
+
print("Creating default admin user...")
|
|
81
|
+
admin_user = User(
|
|
82
|
+
username='admin',
|
|
83
|
+
password=bcrypt.hashpw("admin".encode('utf-8'), bcrypt.gensalt()),
|
|
84
|
+
)
|
|
85
|
+
db.session.add(admin_user)
|
|
86
|
+
db.session.commit()
|
|
87
|
+
else:
|
|
88
|
+
print("Admin user already exists.")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def create_app(config_class=None):
|
|
92
|
+
"""
|
|
93
|
+
create app, init database
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
app.config.from_object(config_class or 'config.get_config()')
|
|
97
|
+
os.makedirs(app.config["OUTPUT_FOLDER"], exist_ok=True)
|
|
98
|
+
# Initialize extensions
|
|
99
|
+
socketio.init_app(app, cors_allowed_origins="*", cookie=None)
|
|
100
|
+
login_manager.init_app(app)
|
|
101
|
+
login_manager.login_view = "auth.login"
|
|
102
|
+
db.init_app(app)
|
|
103
|
+
|
|
104
|
+
# Create database tables
|
|
105
|
+
with app.app_context():
|
|
106
|
+
# db.create_all()
|
|
107
|
+
reset_old_schema(db.engine, app.config['OUTPUT_FOLDER'])
|
|
108
|
+
create_admin()
|
|
109
|
+
|
|
110
|
+
# Additional setup
|
|
111
|
+
utils.create_gui_dir(app.config['OUTPUT_FOLDER'])
|
|
112
|
+
|
|
113
|
+
# logger_list = app.config["LOGGERS"]
|
|
114
|
+
logger_path = os.path.join(app.config["OUTPUT_FOLDER"], app.config["LOGGERS_PATH"])
|
|
115
|
+
logger = utils.start_logger(socketio, 'gui_logger', logger_path)
|
|
116
|
+
|
|
117
|
+
@app.before_request
|
|
118
|
+
def before_request():
|
|
119
|
+
"""
|
|
120
|
+
Called before
|
|
121
|
+
|
|
122
|
+
"""
|
|
123
|
+
g.logger = logger
|
|
124
|
+
g.socketio = socketio
|
|
125
|
+
session.permanent = False
|
|
126
|
+
# DEMO_MODE: Simulate logged-in user per session
|
|
127
|
+
if app.config.get("DEMO_MODE", False):
|
|
128
|
+
if "demo_user_id" not in session:
|
|
129
|
+
session["demo_user_id"] = f"demo_{str(uuid.uuid4())[:8]}"
|
|
130
|
+
|
|
131
|
+
class SessionDemoUser(AnonymousUserMixin):
|
|
132
|
+
@property
|
|
133
|
+
def is_authenticated(self):
|
|
134
|
+
return True
|
|
135
|
+
|
|
136
|
+
def get_id(self):
|
|
137
|
+
return session.get("demo_user_id")
|
|
138
|
+
|
|
139
|
+
login_manager.anonymous_user = SessionDemoUser
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@app.route('/')
|
|
144
|
+
def redirect_to_prefix():
|
|
145
|
+
return redirect(url_for('main.index', version=ivoryos_version)) # Assuming 'index' is a route in your blueprint
|
|
146
|
+
|
|
147
|
+
@app.template_filter('format_name')
|
|
148
|
+
def format_name(name):
|
|
149
|
+
name = name.split(".")[-1]
|
|
150
|
+
text = ' '.join(word for word in name.split('_'))
|
|
151
|
+
return text.capitalize()
|
|
152
|
+
|
|
153
|
+
# app.config.setdefault("DEMO_MODE", False)
|
|
154
|
+
return app
|
ivoryos/config.py
CHANGED
|
@@ -43,6 +43,7 @@ class TestingConfig(Config):
|
|
|
43
43
|
|
|
44
44
|
class DemoConfig(Config):
|
|
45
45
|
DEBUG = False
|
|
46
|
+
DEMO_MODE = True
|
|
46
47
|
SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
|
|
47
48
|
OUTPUT_FOLDER = os.path.join(os.path.abspath(os.curdir), '/tmp/ivoryos_data')
|
|
48
49
|
CSV_FOLDER = os.path.join(OUTPUT_FOLDER, 'config_csv/')
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# optimizers/ax_optimizer.py
|
|
2
|
+
from typing import Dict
|
|
3
|
+
|
|
4
|
+
from pandas import DataFrame
|
|
5
|
+
|
|
6
|
+
from ivoryos.optimizer.base_optimizer import OptimizerBase
|
|
7
|
+
from ivoryos.utils.utils import install_and_import
|
|
8
|
+
|
|
9
|
+
class AxOptimizer(OptimizerBase):
|
|
10
|
+
def __init__(self, experiment_name, parameter_space, objective_config, optimizer_config=None,
|
|
11
|
+
parameter_constraints:list=None, datapath=None):
|
|
12
|
+
self.trial_index_list = None
|
|
13
|
+
try:
|
|
14
|
+
from ax.api.client import Client
|
|
15
|
+
except ImportError as e:
|
|
16
|
+
install_and_import("ax", "ax-platform")
|
|
17
|
+
raise ImportError("Please install Ax with pip install ax-platform to use AxOptimizer. Attempting to install Ax...")
|
|
18
|
+
super().__init__(experiment_name, parameter_space, objective_config, optimizer_config, parameter_constraints, )
|
|
19
|
+
|
|
20
|
+
self.client = Client()
|
|
21
|
+
# 2. Configure where Ax will search.
|
|
22
|
+
self.client.configure_experiment(
|
|
23
|
+
name=experiment_name,
|
|
24
|
+
parameters=self._convert_parameter_to_ax_format(parameter_space),
|
|
25
|
+
parameter_constraints=parameter_constraints
|
|
26
|
+
)
|
|
27
|
+
# 3. Configure the objective function.
|
|
28
|
+
self.client.configure_optimization(objective=self._convert_objective_to_ax_format(objective_config))
|
|
29
|
+
if optimizer_config:
|
|
30
|
+
self.client.set_generation_strategy(self._convert_generator_to_ax_format(optimizer_config))
|
|
31
|
+
self.generators = self._create_generator_mapping()
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def _create_generator_mapping():
|
|
35
|
+
"""Create a mapping from string values to Generator enum members."""
|
|
36
|
+
from ax.adapter import Generators
|
|
37
|
+
return {member.value: member for member in Generators}
|
|
38
|
+
|
|
39
|
+
def _convert_parameter_to_ax_format(self, parameter_space):
|
|
40
|
+
"""
|
|
41
|
+
Converts the parameter space configuration to Baybe format.
|
|
42
|
+
:param parameter_space: The parameter space configuration.
|
|
43
|
+
[
|
|
44
|
+
{"name": "param_1", "type": "range", "bounds": [1.0, 2.0], "value_type": "float"},
|
|
45
|
+
{"name": "param_2", "type": "choice", "bounds": ["a", "b", "c"], "value_type": "str"},
|
|
46
|
+
{"name": "param_3", "type": "range", "bounds": [0 10], "value_type": "int"},
|
|
47
|
+
]
|
|
48
|
+
:return: A list of Baybe parameters.
|
|
49
|
+
"""
|
|
50
|
+
from ax import RangeParameterConfig, ChoiceParameterConfig
|
|
51
|
+
ax_params = []
|
|
52
|
+
for p in parameter_space:
|
|
53
|
+
if p["type"] == "range":
|
|
54
|
+
# if step is used here, convert to ChoiceParameterConfig
|
|
55
|
+
if len(p["bounds"]) == 3:
|
|
56
|
+
values = self._create_discrete_search_space(range_with_step=p["bounds"],value_type=p["value_type"])
|
|
57
|
+
ax_params.append(ChoiceParameterConfig(name=p["name"], values=values, parameter_type="int", is_ordered=True))
|
|
58
|
+
else:
|
|
59
|
+
ax_params.append(
|
|
60
|
+
RangeParameterConfig(
|
|
61
|
+
name=p["name"],
|
|
62
|
+
bounds=tuple(p["bounds"]),
|
|
63
|
+
parameter_type=p["value_type"]
|
|
64
|
+
))
|
|
65
|
+
elif p["type"] == "choice":
|
|
66
|
+
ax_params.append(
|
|
67
|
+
ChoiceParameterConfig(
|
|
68
|
+
name=p["name"],
|
|
69
|
+
values=p["bounds"],
|
|
70
|
+
parameter_type=p["value_type"],
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
return ax_params
|
|
74
|
+
|
|
75
|
+
def _convert_objective_to_ax_format(self, objective_config: list):
|
|
76
|
+
"""
|
|
77
|
+
Converts the objective configuration to Baybe format.
|
|
78
|
+
:param parameter_space: The parameter space configuration.
|
|
79
|
+
[
|
|
80
|
+
{"name": "obj_1", "minimize": True, "weight": 1},
|
|
81
|
+
{"name": "obj_2", "minimize": False, "weight": 2}
|
|
82
|
+
]
|
|
83
|
+
:return: Ax objective configuration. "-cost, utility"
|
|
84
|
+
"""
|
|
85
|
+
objectives = []
|
|
86
|
+
for obj in objective_config:
|
|
87
|
+
obj_name = obj.get("name")
|
|
88
|
+
|
|
89
|
+
# # fixing unknown Ax "unsupported operand type(s) for *: 'One' and 'LazyFunction'" in v1.1.2, test is not allowed as objective name
|
|
90
|
+
if obj_name == "test":
|
|
91
|
+
raise ValueError("test is not allowed as objective name")
|
|
92
|
+
|
|
93
|
+
minimize = obj.get("minimize", True)
|
|
94
|
+
weight = obj.get("weight", 1)
|
|
95
|
+
sign = "-" if minimize else ""
|
|
96
|
+
objectives.append(f"{sign}{weight} * {obj_name}")
|
|
97
|
+
return ", ".join(objectives)
|
|
98
|
+
|
|
99
|
+
def _convert_generator_to_ax_format(self, optimizer_config):
|
|
100
|
+
"""
|
|
101
|
+
Converts the optimizer configuration to Ax format.
|
|
102
|
+
:param optimizer_config: The optimizer configuration.
|
|
103
|
+
:return: Ax generator configuration.
|
|
104
|
+
"""
|
|
105
|
+
from ax.generation_strategy.generation_node import GenerationStep
|
|
106
|
+
from ax.generation_strategy.generation_strategy import GenerationStrategy
|
|
107
|
+
generators = self._create_generator_mapping()
|
|
108
|
+
step_1 = optimizer_config.get("step_1", {})
|
|
109
|
+
step_2 = optimizer_config.get("step_2", {})
|
|
110
|
+
step_1_generator = step_1.get("model", "Sobol")
|
|
111
|
+
step_2_generator = step_2.get("model", "BOTorch")
|
|
112
|
+
generator_1 = GenerationStep(generator=generators.get(step_1_generator), num_trials=step_1.get("num_samples", 5))
|
|
113
|
+
generator_2 = GenerationStep(generator=generators.get(step_2_generator), num_trials=step_2.get("num_samples", -1))
|
|
114
|
+
return GenerationStrategy(steps=[generator_1, generator_2])
|
|
115
|
+
|
|
116
|
+
def suggest(self, n=1):
|
|
117
|
+
trials = self.client.get_next_trials(n)
|
|
118
|
+
trial_index_list = []
|
|
119
|
+
param_list = []
|
|
120
|
+
for trial_index, params in trials.items():
|
|
121
|
+
trial_index_list.append(trial_index)
|
|
122
|
+
param_list.append(params)
|
|
123
|
+
self.trial_index_list = trial_index_list
|
|
124
|
+
return param_list
|
|
125
|
+
|
|
126
|
+
def observe(self, results):
|
|
127
|
+
for trial_index, result in zip(self.trial_index_list, results):
|
|
128
|
+
obj_only_result = {k: v for k, v in result.items() if k in [obj["name"] for obj in self.objective_config]}
|
|
129
|
+
self.client.complete_trial(
|
|
130
|
+
trial_index=trial_index,
|
|
131
|
+
raw_data=obj_only_result
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def get_plots(self, plot_type):
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
def get_schema():
|
|
139
|
+
return {
|
|
140
|
+
"parameter_types": ["range", "choice"],
|
|
141
|
+
"multiple_objectives": True,
|
|
142
|
+
# "objective_weights": True,
|
|
143
|
+
"supports_continuous": True,
|
|
144
|
+
"supports_constraints": True,
|
|
145
|
+
"optimizer_config": {
|
|
146
|
+
"step_1": {"model": ["Sobol", "Uniform", "Factorial", "Thompson"], "num_samples": 5},
|
|
147
|
+
"step_2": {"model": ["BoTorch", "SAASBO", "SAAS_MTGP", "Legacy_GPEI", "EB", "EB_Ashr", "ST_MTGP", "BO_MIXED", "Contextual_SACBO"]}
|
|
148
|
+
},
|
|
149
|
+
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
def append_existing_data(self, existing_data:DataFrame):
|
|
153
|
+
"""
|
|
154
|
+
Append existing data to the Ax experiment.
|
|
155
|
+
:param existing_data: A dictionary containing existing data.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
if isinstance(existing_data, DataFrame):
|
|
159
|
+
if existing_data.empty:
|
|
160
|
+
return
|
|
161
|
+
existing_data = existing_data.to_dict(orient="records")
|
|
162
|
+
parameter_names = [i.get("name") for i in self.parameter_space]
|
|
163
|
+
objective_names = [i.get("name") for i in self.objective_config]
|
|
164
|
+
for entry in existing_data:
|
|
165
|
+
# for name, value in entry.items():
|
|
166
|
+
# First attach the trial and note the trial index
|
|
167
|
+
parameters = {name: value for name, value in entry.items() if name in parameter_names}
|
|
168
|
+
trial_index = self.client.attach_trial(parameters=parameters)
|
|
169
|
+
raw_data = {name: value for name, value in entry.items() if name in objective_names}
|
|
170
|
+
# Then complete the trial with the existing data
|
|
171
|
+
self.client.complete_trial(trial_index=trial_index, raw_data=raw_data)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
if __name__ == "__main__":
|
|
175
|
+
# Example usage
|
|
176
|
+
optimizer = AxOptimizer(
|
|
177
|
+
experiment_name="example_experiment",
|
|
178
|
+
parameter_space=[
|
|
179
|
+
{"name": "param_1", "type": "range", "bounds": [0.0, 1.0], "value_type": "float"},
|
|
180
|
+
{"name": "param_2", "type": "choice", "bounds": ["a", "b", "c"], "value_type": "str"}
|
|
181
|
+
],
|
|
182
|
+
objective_config=[
|
|
183
|
+
{"name": "objective_1", "minimize": True},
|
|
184
|
+
{"name": "objective_2", "minimize": False}
|
|
185
|
+
],
|
|
186
|
+
optimizer_config={
|
|
187
|
+
"step_1": {"model": "Sobol", "num_samples": 5},
|
|
188
|
+
"step_2": {"model": "BoTorch"}
|
|
189
|
+
}
|
|
190
|
+
)
|
|
191
|
+
print(optimizer._create_generator_mapping())
|