ivoryos 1.0.9__py3-none-any.whl → 1.2.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ivoryos might be problematic. Click here for more details.
- ivoryos/__init__.py +26 -7
- ivoryos/routes/api/api.py +56 -0
- ivoryos/routes/auth/auth.py +5 -5
- ivoryos/routes/control/control.py +77 -372
- ivoryos/routes/control/control_file.py +36 -0
- ivoryos/routes/control/control_new_device.py +142 -0
- ivoryos/routes/control/templates/controllers.html +166 -0
- ivoryos/routes/control/templates/controllers_new.html +112 -0
- ivoryos/routes/control/utils.py +38 -0
- ivoryos/routes/data/data.py +129 -0
- ivoryos/routes/data/templates/components/step_card.html +13 -0
- ivoryos/routes/{database/templates/database → data/templates}/workflow_database.html +14 -8
- ivoryos/routes/{database/templates/database → data/templates}/workflow_view.html +3 -3
- ivoryos/routes/design/__init__.py +4 -0
- ivoryos/routes/design/design.py +298 -656
- ivoryos/routes/design/design_file.py +68 -0
- ivoryos/routes/design/design_step.py +145 -0
- ivoryos/routes/design/templates/components/action_form.html +53 -0
- ivoryos/routes/design/templates/components/actions_panel.html +25 -0
- ivoryos/routes/design/templates/components/autofill_toggle.html +10 -0
- ivoryos/routes/design/templates/components/canvas.html +5 -0
- ivoryos/routes/design/templates/components/canvas_footer.html +9 -0
- ivoryos/routes/design/templates/components/canvas_header.html +75 -0
- ivoryos/routes/design/templates/components/canvas_main.html +34 -0
- ivoryos/routes/design/templates/components/deck_selector.html +10 -0
- ivoryos/routes/design/templates/components/edit_action_form.html +38 -0
- ivoryos/routes/design/templates/components/instruments_panel.html +66 -0
- ivoryos/routes/design/templates/components/modals/drop_modal.html +17 -0
- ivoryos/routes/design/templates/components/modals/json_modal.html +22 -0
- ivoryos/routes/design/templates/components/modals/new_script_modal.html +17 -0
- ivoryos/routes/design/templates/components/modals/rename_modal.html +23 -0
- ivoryos/routes/design/templates/components/modals/saveas_modal.html +27 -0
- ivoryos/routes/design/templates/components/modals.html +6 -0
- ivoryos/routes/design/templates/components/python_code_overlay.html +39 -0
- ivoryos/routes/design/templates/components/sidebar.html +15 -0
- ivoryos/routes/design/templates/components/text_to_code_panel.html +20 -0
- ivoryos/routes/design/templates/experiment_builder.html +41 -0
- ivoryos/routes/execute/__init__.py +0 -0
- ivoryos/routes/execute/execute.py +317 -0
- ivoryos/routes/execute/execute_file.py +78 -0
- ivoryos/routes/execute/templates/components/error_modal.html +20 -0
- ivoryos/routes/execute/templates/components/logging_panel.html +31 -0
- ivoryos/routes/execute/templates/components/progress_panel.html +27 -0
- ivoryos/routes/execute/templates/components/run_panel.html +9 -0
- ivoryos/routes/execute/templates/components/run_tabs.html +17 -0
- ivoryos/routes/execute/templates/components/tab_bayesian.html +399 -0
- ivoryos/routes/execute/templates/components/tab_configuration.html +98 -0
- ivoryos/routes/execute/templates/components/tab_repeat.html +14 -0
- ivoryos/routes/execute/templates/experiment_run.html +294 -0
- ivoryos/routes/library/__init__.py +0 -0
- ivoryos/routes/library/library.py +159 -0
- ivoryos/routes/{database/templates/database/scripts_database.html → library/templates/library.html} +30 -22
- ivoryos/routes/main/main.py +1 -1
- ivoryos/routes/main/templates/{main/home.html → home.html} +4 -4
- ivoryos/socket_handlers.py +52 -0
- ivoryos/static/js/action_handlers.js +213 -0
- ivoryos/static/js/db_delete.js +23 -0
- ivoryos/static/js/script_metadata.js +39 -0
- ivoryos/static/js/sortable_design.js +89 -56
- ivoryos/static/js/ui_state.js +113 -0
- ivoryos/templates/base.html +4 -4
- ivoryos/utils/bo_campaign.py +179 -3
- ivoryos/utils/db_models.py +14 -5
- ivoryos/utils/form.py +5 -9
- ivoryos/utils/global_config.py +13 -1
- ivoryos/utils/py_to_json.py +225 -0
- ivoryos/utils/script_runner.py +49 -7
- ivoryos/utils/serilize.py +203 -0
- ivoryos/utils/task_runner.py +4 -1
- ivoryos/version.py +1 -1
- {ivoryos-1.0.9.dist-info → ivoryos-1.2.0b1.dist-info}/METADATA +5 -8
- ivoryos-1.2.0b1.dist-info/RECORD +105 -0
- ivoryos/routes/control/templates/control/controllers.html +0 -78
- ivoryos/routes/control/templates/control/controllers_home.html +0 -55
- ivoryos/routes/control/templates/control/controllers_new.html +0 -89
- ivoryos/routes/database/database.py +0 -306
- ivoryos/routes/database/templates/database/step_card.html +0 -7
- ivoryos/routes/design/templates/design/experiment_builder.html +0 -521
- ivoryos/routes/design/templates/design/experiment_run.html +0 -558
- ivoryos-1.0.9.dist-info/RECORD +0 -61
- /ivoryos/routes/auth/templates/{auth/login.html → login.html} +0 -0
- /ivoryos/routes/auth/templates/{auth/signup.html → signup.html} +0 -0
- /ivoryos/routes/{database → data}/__init__.py +0 -0
- /ivoryos/routes/main/templates/{main/help.html → help.html} +0 -0
- {ivoryos-1.0.9.dist-info → ivoryos-1.2.0b1.dist-info}/LICENSE +0 -0
- {ivoryos-1.0.9.dist-info → ivoryos-1.2.0b1.dist-info}/WHEEL +0 -0
- {ivoryos-1.0.9.dist-info → ivoryos-1.2.0b1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{# Save as modal component #}
|
|
2
|
+
<div class="modal fade" id="saveasModal" tabindex="-1" aria-labelledby="saveasModal" aria-hidden="true" >
|
|
3
|
+
<div class="modal-dialog">
|
|
4
|
+
<div class="modal-content">
|
|
5
|
+
<div class="modal-header">
|
|
6
|
+
<h1 class="modal-title fs-5" id="saveasModal">Save your script as </h1>
|
|
7
|
+
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
8
|
+
</div>
|
|
9
|
+
<form method="POST" name="save-as" action="{{ url_for('library.save_as') }}">
|
|
10
|
+
<div class="modal-body">
|
|
11
|
+
<div class="input-group mb-3">
|
|
12
|
+
<label class="input-group-text" for="run_name">Run Name</label>
|
|
13
|
+
<input class="form-control" type="text" name="run_name" id="run_name" placeholder="{{script['name']}}" required="required">
|
|
14
|
+
</div>
|
|
15
|
+
<div class="form-check form-switch">
|
|
16
|
+
<input class="form-check-input" type="checkbox" name="register_workflow" id="register_workflow">
|
|
17
|
+
<label class="input-group-label" for="register_workflow">Register this workflow</label>
|
|
18
|
+
</div>
|
|
19
|
+
</div>
|
|
20
|
+
<div class="modal-footer">
|
|
21
|
+
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal"> Close </button>
|
|
22
|
+
<button type="submit" class="btn btn-primary" > Save </button>
|
|
23
|
+
</div>
|
|
24
|
+
</form>
|
|
25
|
+
</div>
|
|
26
|
+
</div>
|
|
27
|
+
</div>
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{# Modals component for experiment builder #}
|
|
2
|
+
{% include 'components/modals/new_script_modal.html' %}
|
|
3
|
+
{% include 'components/modals/saveas_modal.html' %}
|
|
4
|
+
{% include 'components/modals/rename_modal.html' %}
|
|
5
|
+
{% include 'components/modals/json_modal.html' %}
|
|
6
|
+
{% include 'components/modals/drop_modal.html' %}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{# Python code overlay component #}
|
|
2
|
+
{#{% if session.get('show_code') %}#}
|
|
3
|
+
<style>
|
|
4
|
+
.code-overlay {
|
|
5
|
+
position: fixed;
|
|
6
|
+
right: 0;
|
|
7
|
+
top: 180px;
|
|
8
|
+
height: 100vh;
|
|
9
|
+
width: 400px;
|
|
10
|
+
z-index: 1000;
|
|
11
|
+
transition: transform 0.3s ease;
|
|
12
|
+
transform: translateX(100%);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
.code-overlay.show {
|
|
16
|
+
transform: translateX(0);
|
|
17
|
+
}
|
|
18
|
+
</style>
|
|
19
|
+
<script>hljs.highlightAll();</script>
|
|
20
|
+
{% if session.get('show_code') %}
|
|
21
|
+
<div id="pythonCodeOverlay" class="code-overlay bg-light border-start show">
|
|
22
|
+
{% else %}
|
|
23
|
+
<div id="pythonCodeOverlay" class="code-overlay bg-light border-start">
|
|
24
|
+
{% endif %}
|
|
25
|
+
<div class="overlay-header d-flex justify-content-between align-items-center px-3 py-2 border-bottom">
|
|
26
|
+
<strong>Python Code</strong>
|
|
27
|
+
<button class="btn btn-sm btn-outline-secondary" onclick="toggleCodeOverlay()">
|
|
28
|
+
<i class="bi bi-x-lg"></i>
|
|
29
|
+
</button>
|
|
30
|
+
</div>
|
|
31
|
+
<div class="overlay-content p-3">
|
|
32
|
+
{% for stype, script in session.get('python_code', {}).items() %}
|
|
33
|
+
<pre><code class="language-python">{{ script }}</code></pre>
|
|
34
|
+
{% endfor %}
|
|
35
|
+
<a href="{{ url_for('design.design_files.download_python', filetype='python') }}">
|
|
36
|
+
Download <i class="bi bi-download"></i>
|
|
37
|
+
</a>
|
|
38
|
+
</div>
|
|
39
|
+
</div>
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{# Sidebar component for experiment builder #}
|
|
2
|
+
<div class="instrument-panel" id="instrument-panel">
|
|
3
|
+
{% if instrument %}
|
|
4
|
+
<div class="instrument-methods" id="instrument-methods">
|
|
5
|
+
{% include 'components/actions_panel.html' %}
|
|
6
|
+
</div>
|
|
7
|
+
{% else %}
|
|
8
|
+
{# select deck if this is online#}
|
|
9
|
+
{% if off_line %}
|
|
10
|
+
{% include 'components/deck_selector.html' %}
|
|
11
|
+
{% endif %}
|
|
12
|
+
|
|
13
|
+
{% include 'components/instruments_panel.html' %}
|
|
14
|
+
{% endif %}
|
|
15
|
+
</div>
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{# Text-to-code panel component #}
|
|
2
|
+
<div class="accordion-item text-to-code">
|
|
3
|
+
<h2 class="accordion-header">
|
|
4
|
+
<button class="accordion-button text-to-code" type="button" data-bs-toggle="collapse" data-bs-target="#text-to-code" aria-expanded="false" aria-controls="collapseExample">
|
|
5
|
+
Text-to-Code
|
|
6
|
+
</button>
|
|
7
|
+
</h2>
|
|
8
|
+
<div id="text-to-code" class="accordion-collapse collapse show" data-bs-parent="#accordionActions">
|
|
9
|
+
<div class="accordion-body">
|
|
10
|
+
<form role="form" method='POST' name="generate" id="generate" action="{{url_for('design.generate_code')}}">
|
|
11
|
+
<input type="hidden" id="instrument" name="instrument" value="{{instrument}}">
|
|
12
|
+
<textarea class="form-control" id="prompt" name="prompt" rows="6" aria-describedby="promptHelpBlock">{{ session['prompt'][instrument] if instrument in session['prompt'] else '' }}</textarea>
|
|
13
|
+
<div id="promptHelpBlock" class="form-text">
|
|
14
|
+
This will overwrite current design.
|
|
15
|
+
</div>
|
|
16
|
+
<button type="submit" class="btn btn-dark" id="gen" name="gen">Generate</button>
|
|
17
|
+
</form>
|
|
18
|
+
</div>
|
|
19
|
+
</div>
|
|
20
|
+
</div>
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{% extends 'base.html' %}
|
|
2
|
+
{% block title %}IvoryOS | Design{% endblock %}
|
|
3
|
+
|
|
4
|
+
{% block body %}
|
|
5
|
+
{# overlay block for text-to-code gen #}
|
|
6
|
+
|
|
7
|
+
<div id="overlay" class="overlay">
|
|
8
|
+
<div>
|
|
9
|
+
<h3 id="overlay-text">Generating design, please wait...</h3>
|
|
10
|
+
<div class="spinner-border" role="status"></div>
|
|
11
|
+
</div>
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
<div class="row">
|
|
15
|
+
<div class="col-md-3 scroll-column" id="sidebar-wrapper">
|
|
16
|
+
{% include 'components/sidebar.html' %}
|
|
17
|
+
</div>
|
|
18
|
+
<div class="col-md-9 scroll-column" id="canvas-wrapper">
|
|
19
|
+
{% include 'components/canvas.html' %}
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
</div>
|
|
23
|
+
|
|
24
|
+
{# Include all modals #}
|
|
25
|
+
{% include 'components/modals.html' %}
|
|
26
|
+
|
|
27
|
+
{# Include all scripts #}
|
|
28
|
+
<script>
|
|
29
|
+
const updateListUrl = "{{ url_for('design.design_steps.update_list') }}";
|
|
30
|
+
const scriptUIStateUrl = "{{ url_for('design.update_ui_state') }}";
|
|
31
|
+
const scriptMetaUrl = "{{ url_for('design.update_script_meta') }}";
|
|
32
|
+
const scriptStepUrl = `{{ url_for('design.design_steps.get_step', uuid=0) }}`;
|
|
33
|
+
const scriptStepDupUrl = `{{ url_for('design.design_steps.duplicate_action', uuid=0) }}`;
|
|
34
|
+
const scriptDeleteUrl = "{{ url_for('design.clear_draft') }}";
|
|
35
|
+
</script>
|
|
36
|
+
<script src="{{ url_for('static', filename='js/sortable_design.js') }}"></script>
|
|
37
|
+
<script src="{{ url_for('static', filename='js/action_handlers.js') }}"></script>
|
|
38
|
+
<script src="{{ url_for('static', filename='js/script_metadata.js') }}"></script>
|
|
39
|
+
<script src="{{ url_for('static', filename='js/ui_state.js') }}"></script>
|
|
40
|
+
|
|
41
|
+
{% endblock %}
|
|
File without changes
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
from flask import Blueprint, redirect, url_for, flash, jsonify, request, render_template, session, \
|
|
6
|
+
current_app, g
|
|
7
|
+
from flask_login import login_required
|
|
8
|
+
|
|
9
|
+
from ivoryos.routes.execute.execute_file import files
|
|
10
|
+
from ivoryos.utils import utils
|
|
11
|
+
from ivoryos.utils.bo_campaign import parse_optimization_form
|
|
12
|
+
from ivoryos.utils.db_models import SingleStep, WorkflowRun, WorkflowStep
|
|
13
|
+
from ivoryos.utils.global_config import GlobalConfig
|
|
14
|
+
from ivoryos.utils.form import create_action_button
|
|
15
|
+
|
|
16
|
+
from werkzeug.utils import secure_filename
|
|
17
|
+
|
|
18
|
+
from ivoryos.socket_handlers import runner, retry, pause, abort_pending, abort_current
|
|
19
|
+
|
|
20
|
+
execute = Blueprint('execute', __name__, template_folder='templates')
|
|
21
|
+
|
|
22
|
+
execute.register_blueprint(files)
|
|
23
|
+
# Register sub-blueprints
|
|
24
|
+
global_config = GlobalConfig()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@execute.route("/executions/config", methods=['GET', 'POST'])
|
|
28
|
+
@login_required
|
|
29
|
+
def experiment_run():
|
|
30
|
+
"""
|
|
31
|
+
.. :quickref: Workflow Execution Config; Execute/iterate the workflow
|
|
32
|
+
|
|
33
|
+
.. http:get:: /executions/config
|
|
34
|
+
|
|
35
|
+
Load the experiment execution interface.
|
|
36
|
+
|
|
37
|
+
.. http:post:: /executions/config
|
|
38
|
+
|
|
39
|
+
Start workflow execution with experiment configuration.
|
|
40
|
+
|
|
41
|
+
"""
|
|
42
|
+
deck = global_config.deck
|
|
43
|
+
script = utils.get_script_file()
|
|
44
|
+
# runner = global_config.runner
|
|
45
|
+
existing_data = None
|
|
46
|
+
# script.sort_actions() # handled in update list
|
|
47
|
+
off_line = current_app.config["OFF_LINE"]
|
|
48
|
+
deck_list = utils.import_history(os.path.join(current_app.config["OUTPUT_FOLDER"], 'deck_history.txt'))
|
|
49
|
+
optimizers_schema = {k: v.get_schema() for k, v in global_config.optimizers.items()}
|
|
50
|
+
design_buttons = {stype: create_action_button(script, stype) for stype in script.stypes}
|
|
51
|
+
config_preview = []
|
|
52
|
+
config_file_list = [i for i in os.listdir(current_app.config["CSV_FOLDER"]) if not i == ".gitkeep"]
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
exec_string = script.python_script if script.python_script else script.compile(
|
|
56
|
+
current_app.config['SCRIPT_FOLDER'])
|
|
57
|
+
except Exception as e:
|
|
58
|
+
flash(e.__str__())
|
|
59
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
60
|
+
return jsonify({"error": e.__str__()})
|
|
61
|
+
else:
|
|
62
|
+
return redirect(url_for("design.experiment_builder"))
|
|
63
|
+
|
|
64
|
+
config_file = request.args.get("filename")
|
|
65
|
+
config = []
|
|
66
|
+
if config_file:
|
|
67
|
+
session['config_file'] = config_file
|
|
68
|
+
filename = session.get("config_file")
|
|
69
|
+
if filename:
|
|
70
|
+
config = list(csv.DictReader(open(os.path.join(current_app.config['CSV_FOLDER'], filename))))
|
|
71
|
+
config_preview = config[1:]
|
|
72
|
+
arg_type = config.pop(0) # first entry is types
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
# Handle both string and dict exec_string
|
|
76
|
+
if isinstance(exec_string, dict):
|
|
77
|
+
for key, func_str in exec_string.items():
|
|
78
|
+
exec(func_str)
|
|
79
|
+
line_collection = script.convert_to_lines(exec_string)
|
|
80
|
+
else:
|
|
81
|
+
# Handle string case - you might need to adjust this based on your needs
|
|
82
|
+
line_collection = []
|
|
83
|
+
except Exception:
|
|
84
|
+
flash(f"Please check syntax!!")
|
|
85
|
+
return redirect(url_for("design.experiment_builder"))
|
|
86
|
+
|
|
87
|
+
run_name = script.name if script.name else "untitled"
|
|
88
|
+
|
|
89
|
+
dismiss = session.get("dismiss", None)
|
|
90
|
+
script = utils.get_script_file()
|
|
91
|
+
no_deck_warning = False
|
|
92
|
+
|
|
93
|
+
_, return_list = script.config_return()
|
|
94
|
+
config_list, config_type_list = script.config("script")
|
|
95
|
+
data_list = os.listdir(current_app.config['DATA_FOLDER'])
|
|
96
|
+
data_list.remove(".gitkeep") if ".gitkeep" in data_list else data_list
|
|
97
|
+
|
|
98
|
+
if deck is None:
|
|
99
|
+
no_deck_warning = True
|
|
100
|
+
flash(f"No deck is found, import {script.deck}")
|
|
101
|
+
elif script.deck:
|
|
102
|
+
is_deck_match = script.deck == deck.__name__ or script.deck == \
|
|
103
|
+
os.path.splitext(os.path.basename(deck.__file__))[0]
|
|
104
|
+
if not is_deck_match:
|
|
105
|
+
flash(f"This script is not compatible with current deck, import {script.deck}")
|
|
106
|
+
|
|
107
|
+
if request.method == "POST":
|
|
108
|
+
bo_args = None
|
|
109
|
+
compiled = False
|
|
110
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
111
|
+
payload_json = request.get_json()
|
|
112
|
+
compiled = True
|
|
113
|
+
if "kwargs" in payload_json:
|
|
114
|
+
config = payload_json["kwargs"]
|
|
115
|
+
elif "parameters" in payload_json:
|
|
116
|
+
bo_args = payload_json
|
|
117
|
+
repeat = payload_json.pop("repeat", None)
|
|
118
|
+
else:
|
|
119
|
+
if "bo" in request.form:
|
|
120
|
+
bo_args = request.form.to_dict()
|
|
121
|
+
existing_data = bo_args.pop("existing_data")
|
|
122
|
+
if "online-config" in request.form:
|
|
123
|
+
config = utils.web_config_entry_wrapper(request.form.to_dict(), config_list)
|
|
124
|
+
repeat = request.form.get('repeat', None)
|
|
125
|
+
|
|
126
|
+
try:
|
|
127
|
+
datapath = current_app.config["DATA_FOLDER"]
|
|
128
|
+
run_name = script.validate_function_name(run_name)
|
|
129
|
+
runner.run_script(script=script, run_name=run_name, config=config, bo_args=bo_args,
|
|
130
|
+
logger=g.logger, socketio=g.socketio, repeat_count=repeat,
|
|
131
|
+
output_path=datapath, compiled=compiled, history=existing_data,
|
|
132
|
+
current_app=current_app._get_current_object()
|
|
133
|
+
)
|
|
134
|
+
if utils.check_config_duplicate(config):
|
|
135
|
+
flash(f"WARNING: Duplicate in config entries.")
|
|
136
|
+
except Exception as e:
|
|
137
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
138
|
+
return jsonify({"error": e.__str__()})
|
|
139
|
+
else:
|
|
140
|
+
flash(e)
|
|
141
|
+
|
|
142
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
143
|
+
# wait to get a workflow ID
|
|
144
|
+
while not global_config.runner_status:
|
|
145
|
+
time.sleep(1)
|
|
146
|
+
return jsonify({"status": "task started", "task_id": global_config.runner_status.get("id")})
|
|
147
|
+
else:
|
|
148
|
+
return render_template('experiment_run.html', script=script.script_dict, filename=filename,
|
|
149
|
+
dot_py=exec_string, line_collection=line_collection,
|
|
150
|
+
return_list=return_list, config_list=config_list, config_file_list=config_file_list,
|
|
151
|
+
config_preview=config_preview, data_list=data_list, config_type_list=config_type_list,
|
|
152
|
+
no_deck_warning=no_deck_warning, dismiss=dismiss, design_buttons=design_buttons,
|
|
153
|
+
history=deck_list, pause_status=runner.pause_status(), optimizer_schema=optimizers_schema)
|
|
154
|
+
|
|
155
|
+
@execute.route("/executions/campaign", methods=["POST"])
|
|
156
|
+
@login_required
|
|
157
|
+
def run_bo():
|
|
158
|
+
"""
|
|
159
|
+
.. :quickref: Workflow Execution; run Bayesian Optimization
|
|
160
|
+
Run Bayesian Optimization with the given parameters and objectives.
|
|
161
|
+
|
|
162
|
+
.. http:post:: /executions/campaign
|
|
163
|
+
|
|
164
|
+
:form repeat: number of iterations to run
|
|
165
|
+
:form optimizer_type: type of optimizer to use
|
|
166
|
+
:form existing_data: existing data to use for optimization
|
|
167
|
+
:form parameters: parameters for optimization
|
|
168
|
+
:form objectives: objectives for optimization
|
|
169
|
+
TODO: merge to experiment_run or not, add more details about the form fields and their expected values.
|
|
170
|
+
"""
|
|
171
|
+
script = utils.get_script_file()
|
|
172
|
+
run_name = script.name if script.name else "untitled"
|
|
173
|
+
payload = request.form.to_dict()
|
|
174
|
+
repeat = payload.pop("repeat", None)
|
|
175
|
+
optimizer_type = payload.pop("optimizer_type", None)
|
|
176
|
+
existing_data = payload.pop("existing_data", None)
|
|
177
|
+
parameters, objectives, steps = parse_optimization_form(payload)
|
|
178
|
+
try:
|
|
179
|
+
datapath = current_app.config["DATA_FOLDER"]
|
|
180
|
+
run_name = script.validate_function_name(run_name)
|
|
181
|
+
Optimizer = global_config.optimizers.get(optimizer_type, None)
|
|
182
|
+
if not Optimizer:
|
|
183
|
+
raise ValueError(f"Optimizer {optimizer_type} is not supported or not found.")
|
|
184
|
+
optimizer = Optimizer(experiment_name=run_name, parameter_space=parameters, objective_config=objectives,
|
|
185
|
+
optimizer_config=steps)
|
|
186
|
+
runner.run_script(script=script, run_name=run_name, optimizer=optimizer,
|
|
187
|
+
logger=g.logger, socketio=g.socketio, repeat_count=repeat,
|
|
188
|
+
output_path=datapath, compiled=False, history=existing_data,
|
|
189
|
+
current_app=current_app._get_current_object()
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
except Exception as e:
|
|
193
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
194
|
+
return jsonify({"error": e.__str__()})
|
|
195
|
+
else:
|
|
196
|
+
flash(e.__str__())
|
|
197
|
+
return redirect(url_for("execute.experiment_run"))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@execute.route("/executions/status", methods=["GET"])
|
|
202
|
+
def runner_status():
|
|
203
|
+
"""
|
|
204
|
+
.. :quickref: Workflow Execution Control; backend runner status
|
|
205
|
+
|
|
206
|
+
get is system is busy and current task
|
|
207
|
+
|
|
208
|
+
.. http:get:: /executions/status
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
"""
|
|
212
|
+
# runner = global_config.runner
|
|
213
|
+
runner_busy = global_config.runner_lock.locked()
|
|
214
|
+
status = {"busy": runner_busy}
|
|
215
|
+
task_status = global_config.runner_status
|
|
216
|
+
current_step = {}
|
|
217
|
+
|
|
218
|
+
if task_status is not None:
|
|
219
|
+
task_type = task_status["type"]
|
|
220
|
+
task_id = task_status["id"]
|
|
221
|
+
if task_type == "task":
|
|
222
|
+
# todo
|
|
223
|
+
step = SingleStep.query.get(task_id)
|
|
224
|
+
current_step = step.as_dict()
|
|
225
|
+
if task_type == "workflow":
|
|
226
|
+
workflow = WorkflowRun.query.get(task_id)
|
|
227
|
+
if workflow is not None:
|
|
228
|
+
latest_step = WorkflowStep.query.filter_by(workflow_id=workflow.id).order_by(
|
|
229
|
+
WorkflowStep.start_time.desc()).first()
|
|
230
|
+
if latest_step is not None:
|
|
231
|
+
current_step = latest_step.as_dict()
|
|
232
|
+
status["workflow_status"] = {"workflow_info": workflow.as_dict(), "runner_status": runner.get_status()}
|
|
233
|
+
status["current_task"] = current_step
|
|
234
|
+
return jsonify(status), 200
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@execute.route("/executions/abort/next-iteration", methods=["POST"])
|
|
238
|
+
def api_abort_pending():
|
|
239
|
+
"""
|
|
240
|
+
.. :quickref: Workflow Execution control; abort pending workflow
|
|
241
|
+
|
|
242
|
+
finish the current iteration and stop pending workflow iterations
|
|
243
|
+
|
|
244
|
+
.. http:get:: /executions/abort/next-iteration
|
|
245
|
+
|
|
246
|
+
"""
|
|
247
|
+
abort_pending()
|
|
248
|
+
return jsonify({"status": "ok"}), 200
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@execute.route("/executions/abort/next-task", methods=["POST"])
|
|
252
|
+
def api_abort_current():
|
|
253
|
+
"""
|
|
254
|
+
.. :quickref: Workflow Execution Control; abort all pending tasks starting from the next task
|
|
255
|
+
|
|
256
|
+
finish the current task and stop all pending tasks or iterations
|
|
257
|
+
|
|
258
|
+
.. http:get:: /executions/abort/next-task
|
|
259
|
+
|
|
260
|
+
"""
|
|
261
|
+
abort_current()
|
|
262
|
+
return jsonify({"status": "ok"}), 200
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
@execute.route("/executions/pause-resume", methods=["POST"])
|
|
266
|
+
def api_pause():
|
|
267
|
+
"""
|
|
268
|
+
.. :quickref: Workflow Execution Control; pause and resume
|
|
269
|
+
|
|
270
|
+
pause workflow iterations or resume workflow iterations
|
|
271
|
+
|
|
272
|
+
.. http:get:: /executions/pause-resume
|
|
273
|
+
|
|
274
|
+
"""
|
|
275
|
+
msg = pause()
|
|
276
|
+
return jsonify({"status": "ok", "pause_status": msg}), 200
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@execute.route("/executions/retry", methods=["POST"])
|
|
280
|
+
def api_retry():
|
|
281
|
+
"""
|
|
282
|
+
.. :quickref: Workflow Execution Control; retry the failed workflow execution step.
|
|
283
|
+
|
|
284
|
+
retry the failed workflow execution step.
|
|
285
|
+
|
|
286
|
+
.. http:get:: /executions/retry
|
|
287
|
+
|
|
288
|
+
"""
|
|
289
|
+
retry()
|
|
290
|
+
return jsonify({"status": "ok, retrying failed step"}), 200
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@execute.route('/files/preview/<string:filename>')
|
|
294
|
+
@login_required
|
|
295
|
+
def data_preview(filename):
|
|
296
|
+
"""
|
|
297
|
+
.. :quickref: Workflow Execution Files; preview a workflow history file (.CSV)
|
|
298
|
+
|
|
299
|
+
Preview the contents of a workflow history file in CSV format.
|
|
300
|
+
|
|
301
|
+
.. http:get:: /files/preview/<str:filename>
|
|
302
|
+
"""
|
|
303
|
+
import csv
|
|
304
|
+
import os
|
|
305
|
+
from flask import abort
|
|
306
|
+
|
|
307
|
+
data_folder = current_app.config['DATA_FOLDER']
|
|
308
|
+
file_path = os.path.join(data_folder, filename)
|
|
309
|
+
if not os.path.exists(file_path):
|
|
310
|
+
abort(404)
|
|
311
|
+
with open(file_path, newline='') as csvfile:
|
|
312
|
+
reader = csv.DictReader(csvfile)
|
|
313
|
+
rows = list(reader)
|
|
314
|
+
# Limit preview to first 10 rows
|
|
315
|
+
return jsonify({"columns": reader.fieldnames, "rows": rows})
|
|
316
|
+
|
|
317
|
+
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from flask import Blueprint, send_file, request, flash, redirect, url_for, session, current_app
|
|
5
|
+
from werkzeug.utils import secure_filename
|
|
6
|
+
from ivoryos.utils import utils
|
|
7
|
+
|
|
8
|
+
files = Blueprint('execute_files', __name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@files.route('/files/execution-configs')
|
|
12
|
+
def download_empty_config():
|
|
13
|
+
"""
|
|
14
|
+
.. :quickref: Workflow Files; download an empty workflow config file (.CSV)
|
|
15
|
+
|
|
16
|
+
.. http:get:: /files/execution-configs
|
|
17
|
+
|
|
18
|
+
:form file: workflow design CSV file
|
|
19
|
+
:status 302: load pseudo deck and then redirects to :http:get:`/ivoryos/executions/config`
|
|
20
|
+
"""
|
|
21
|
+
script = utils.get_script_file()
|
|
22
|
+
run_name = script.name if script.name else "untitled"
|
|
23
|
+
|
|
24
|
+
filepath = os.path.join(current_app.config['SCRIPT_FOLDER'], f"{run_name}_config.csv")
|
|
25
|
+
with open(filepath, 'w', newline='') as f:
|
|
26
|
+
writer = csv.writer(f)
|
|
27
|
+
cfg, cfg_types = script.config("script")
|
|
28
|
+
writer.writerow(cfg)
|
|
29
|
+
writer.writerow(list(cfg_types.values()))
|
|
30
|
+
return send_file(os.path.abspath(filepath), as_attachment=True)
|
|
31
|
+
|
|
32
|
+
@files.route('/files/batch-configs', methods=['POST'])
|
|
33
|
+
def upload():
|
|
34
|
+
"""
|
|
35
|
+
.. :quickref: Workflow Files; upload a workflow config file (.CSV)
|
|
36
|
+
|
|
37
|
+
.. http:post:: /files/execution-configs
|
|
38
|
+
|
|
39
|
+
:form file: workflow CSV config file
|
|
40
|
+
:status 302: save csv file and then redirects to :http:get:`/ivoryos/executions/config`
|
|
41
|
+
"""
|
|
42
|
+
if request.method == "POST":
|
|
43
|
+
f = request.files['file']
|
|
44
|
+
if 'file' not in request.files:
|
|
45
|
+
flash('No file part')
|
|
46
|
+
if f.filename.split('.')[-1] == "csv":
|
|
47
|
+
filename = secure_filename(f.filename)
|
|
48
|
+
f.save(os.path.join(current_app.config['CSV_FOLDER'], filename))
|
|
49
|
+
session['config_file'] = filename
|
|
50
|
+
return redirect(url_for("execute.experiment_run"))
|
|
51
|
+
else:
|
|
52
|
+
flash("Config file is in csv format")
|
|
53
|
+
return redirect(url_for("execute.experiment_run"))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@files.route('/files/execution-data', methods=['POST'])
|
|
57
|
+
def upload_history():
|
|
58
|
+
"""
|
|
59
|
+
.. :quickref: Workflow Files; upload a workflow history file (.CSV)
|
|
60
|
+
|
|
61
|
+
.. http:post:: /files/execution-data
|
|
62
|
+
|
|
63
|
+
:form file: workflow history CSV file
|
|
64
|
+
:status 302: save csv file and then redirects to :http:get:`/ivoryos/executions/config`
|
|
65
|
+
"""
|
|
66
|
+
if request.method == "POST":
|
|
67
|
+
f = request.files['historyfile']
|
|
68
|
+
if 'historyfile' not in request.files:
|
|
69
|
+
flash('No file part')
|
|
70
|
+
if f.filename.split('.')[-1] == "csv":
|
|
71
|
+
filename = secure_filename(f.filename)
|
|
72
|
+
f.save(os.path.join(current_app.config['DATA_FOLDER'], filename))
|
|
73
|
+
return redirect(url_for("execute.experiment_run"))
|
|
74
|
+
else:
|
|
75
|
+
flash("Config file is in csv format")
|
|
76
|
+
return redirect(url_for("execute.experiment_run"))
|
|
77
|
+
|
|
78
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{# Error modal component for experiment run #}
|
|
2
|
+
<div class="modal fade" id="error-modal" tabindex="-1" aria-labelledby="errorModalLabel" aria-hidden="true">
|
|
3
|
+
<div class="modal-dialog">
|
|
4
|
+
<div class="modal-content">
|
|
5
|
+
<div class="modal-header">
|
|
6
|
+
<h5 class="modal-title" id="errorModalLabel">Error Detected</h5>
|
|
7
|
+
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
8
|
+
</div>
|
|
9
|
+
<div class="modal-body">
|
|
10
|
+
<p id="error-message">An error has occurred.</p>
|
|
11
|
+
<p>Do you want to continue execution or stop?</p>
|
|
12
|
+
</div>
|
|
13
|
+
<div class="modal-footer">
|
|
14
|
+
<button type="button" class="btn btn-primary" id="retry-btn" data-bs-dismiss="modal">Rerun Current Step</button>
|
|
15
|
+
<button type="button" class="btn btn-success" id="continue-btn" data-bs-dismiss="modal">Continue</button>
|
|
16
|
+
<button type="button" class="btn btn-danger" id="stop-btn" data-bs-dismiss="modal">Stop Execution</button>
|
|
17
|
+
</div>
|
|
18
|
+
</div>
|
|
19
|
+
</div>
|
|
20
|
+
</div>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{# Logging panel component for experiment run #}
|
|
2
|
+
<div class="col-lg-6 col-sm-12 logging-panel">
|
|
3
|
+
<p>
|
|
4
|
+
<div class="p d-flex justify-content-between align-items-center">
|
|
5
|
+
<h5>Progress:</h5>
|
|
6
|
+
<div class="d-flex gap-2 ms-auto">
|
|
7
|
+
<button id="pause-resume" class="btn btn-info text-white" data-bs-toggle="tooltip" title="Pause execution">
|
|
8
|
+
{% if pause_status %}
|
|
9
|
+
<i class="bi bi-play-circle"></i>
|
|
10
|
+
{% else %}
|
|
11
|
+
<i class="bi bi-pause-circle"></i>
|
|
12
|
+
{% endif %}
|
|
13
|
+
</button>
|
|
14
|
+
<button id="abort-current" class="btn btn-danger text-white" data-bs-toggle="tooltip" title="Stop execution after current step">
|
|
15
|
+
<i class="bi bi-stop-circle"></i>
|
|
16
|
+
</button>
|
|
17
|
+
<button id="abort-pending" class="btn btn-warning text-white" data-bs-toggle="tooltip" title="Stop execution after current iteration">
|
|
18
|
+
<i class="bi bi-hourglass-split"></i>
|
|
19
|
+
</button>
|
|
20
|
+
</div>
|
|
21
|
+
</div>
|
|
22
|
+
<div class="text-muted mt-2">
|
|
23
|
+
<small><strong>Note:</strong> The current step cannot be paused or stopped until it completes. </small>
|
|
24
|
+
</div>
|
|
25
|
+
|
|
26
|
+
<div class="progress" role="progressbar" aria-label="Animated striped example" aria-valuenow="10" aria-valuemin="0" aria-valuemax="100">
|
|
27
|
+
<div id="progress-bar-inner" class="progress-bar progress-bar-striped progress-bar-animated"></div>
|
|
28
|
+
</div>
|
|
29
|
+
<p><h5>Log:</h5></p>
|
|
30
|
+
<div id="logging-panel"></div>
|
|
31
|
+
</div>
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{# Progress panel component for experiment run #}
|
|
2
|
+
<div class="col-lg-6 col-sm-12" id="code-panel" style="{{ '' if pause_status else 'display: none;'}}">
|
|
3
|
+
<p>
|
|
4
|
+
<h5>Progress:</h5>
|
|
5
|
+
{% if "prep" in line_collection.keys() %}
|
|
6
|
+
{% set stype = "prep" %}
|
|
7
|
+
<h6>Preparation:</h6>
|
|
8
|
+
{% for code in line_collection["prep"] %}
|
|
9
|
+
<pre style="margin: 0; padding: 0; line-height: 1;"><code class="python" id="{{ stype }}-{{ loop.index0 }}" >{{code}}</code></pre>
|
|
10
|
+
{% endfor %}
|
|
11
|
+
{% endif %}
|
|
12
|
+
{% if "script" in line_collection.keys() %}
|
|
13
|
+
{% set stype = "script" %}
|
|
14
|
+
<h6>Experiment:</h6>
|
|
15
|
+
{% for code in line_collection["script"] %}
|
|
16
|
+
<pre style="margin: 0; padding: 0; line-height: 1;"><code class="python" id="{{ stype }}-{{ loop.index0 }}" >{{code}}</code></pre>
|
|
17
|
+
{% endfor %}
|
|
18
|
+
{% endif %}
|
|
19
|
+
{% if "cleanup" in line_collection.keys() %}
|
|
20
|
+
{% set stype = "cleanup" %}
|
|
21
|
+
<h6>Cleanup:</h6>
|
|
22
|
+
{% for code in line_collection["cleanup"] %}
|
|
23
|
+
<pre style="margin: 0; padding: 0; line-height: 1;"><code class="python" id="{{ stype }}-{{ loop.index0 }}" >{{code}}</code></pre>
|
|
24
|
+
{% endfor %}
|
|
25
|
+
{% endif %}
|
|
26
|
+
</p>
|
|
27
|
+
</div>
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{# Run panel component for experiment run #}
|
|
2
|
+
{% if script['script'] or script['prep'] or script['cleanup'] %}
|
|
3
|
+
<div class="col-lg-6 col-sm-12" id="run-panel" style="{{ 'display: none;' if pause_status else '' }}">
|
|
4
|
+
{% include 'components/run_tabs.html' %}
|
|
5
|
+
</div>
|
|
6
|
+
{% else %}
|
|
7
|
+
<div class="col-lg-6 col-sm-12" id="placeholder-panel">
|
|
8
|
+
</div>
|
|
9
|
+
{% endif %}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{# Run tabs component for experiment run #}
|
|
2
|
+
<ul class="nav nav-tabs" id="myTabs" role="tablist">
|
|
3
|
+
<li class="nav-item" role="presentation">
|
|
4
|
+
<a class="nav-link {{ 'disabled' if config_list else '' }} {{ 'active' if not config_list else '' }}" id="tab1-tab" data-bs-toggle="tab" href="#tab1" role="tab" aria-controls="tab1" aria-selected="false">Repeat</a>
|
|
5
|
+
</li>
|
|
6
|
+
<li class="nav-item" role="presentation">
|
|
7
|
+
<a class="nav-link {{ 'disabled' if not config_list else '' }} {{ 'active' if config_list else '' }}" id="tab2-tab" data-bs-toggle="tab" href="#tab2" role="tab" aria-controls="tab2" aria-selected="false">Configuration</a>
|
|
8
|
+
</li>
|
|
9
|
+
<li class="nav-item" role="presentation">
|
|
10
|
+
<a class="nav-link {{ 'disabled' if not config_list or not return_list else '' }}" id="tab3-tab" data-bs-toggle="tab" href="#tab3" role="tab" aria-controls="tab3" aria-selected="false">Bayesian Optimization</a>
|
|
11
|
+
</li>
|
|
12
|
+
</ul>
|
|
13
|
+
<div class="tab-content" id="myTabsContent">
|
|
14
|
+
{% include 'components/tab_repeat.html' %}
|
|
15
|
+
{% include 'components/tab_configuration.html' %}
|
|
16
|
+
{% include 'components/tab_bayesian.html' %}
|
|
17
|
+
</div>
|