ivoryos 1.2.8__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ivoryos might be problematic. Click here for more details.
- ivoryos/app.py +38 -1
- ivoryos/routes/api/api.py +2 -1
- ivoryos/routes/control/templates/controllers.html +9 -0
- ivoryos/routes/data/data.py +80 -41
- ivoryos/routes/data/templates/components/step_card.html +42 -13
- ivoryos/routes/data/templates/workflow_view.html +334 -113
- ivoryos/server.py +1 -1
- ivoryos/utils/db_models.py +44 -8
- ivoryos/utils/form.py +4 -2
- ivoryos/utils/script_runner.py +103 -48
- ivoryos/utils/utils.py +6 -3
- ivoryos/version.py +1 -1
- {ivoryos-1.2.8.dist-info → ivoryos-1.3.0.dist-info}/METADATA +1 -1
- {ivoryos-1.2.8.dist-info → ivoryos-1.3.0.dist-info}/RECORD +17 -17
- {ivoryos-1.2.8.dist-info → ivoryos-1.3.0.dist-info}/WHEEL +0 -0
- {ivoryos-1.2.8.dist-info → ivoryos-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {ivoryos-1.2.8.dist-info → ivoryos-1.3.0.dist-info}/top_level.txt +0 -0
ivoryos/app.py
CHANGED
|
@@ -17,6 +17,42 @@ from ivoryos.routes.api.api import api
|
|
|
17
17
|
from ivoryos.socket_handlers import socketio
|
|
18
18
|
from ivoryos.routes.main.main import main
|
|
19
19
|
from ivoryos.version import __version__ as ivoryos_version
|
|
20
|
+
from sqlalchemy import inspect, text
|
|
21
|
+
from flask import current_app
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def reset_old_schema(engine, db_dir):
|
|
25
|
+
inspector = inspect(engine)
|
|
26
|
+
tables = inspector.get_table_names()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# Check if old tables exist (no workflow_phases table)
|
|
30
|
+
has_workflow_phase = 'workflow_phases' in tables
|
|
31
|
+
old_workflow_run = 'old_workflow_run' in tables
|
|
32
|
+
old_workflow_step = 'workflow_steps' in tables
|
|
33
|
+
|
|
34
|
+
if not has_workflow_phase:
|
|
35
|
+
print("⚠️ Old workflow database detected! All previous workflows have been reset to support the new schema.")
|
|
36
|
+
# Backup old DB
|
|
37
|
+
db_path = os.path.join(db_dir, "ivoryos.db")
|
|
38
|
+
if os.path.exists(db_path):
|
|
39
|
+
# os.makedirs(backup_dir, exist_ok=True)
|
|
40
|
+
from datetime import datetime
|
|
41
|
+
import shutil
|
|
42
|
+
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
43
|
+
backup_path = os.path.join(db_dir, f"ivoryos_backup_{ts}.db")
|
|
44
|
+
shutil.copy(db_path, backup_path)
|
|
45
|
+
print(f"Backup created at {backup_path}")
|
|
46
|
+
with engine.begin() as conn:
|
|
47
|
+
# Drop old tables
|
|
48
|
+
if old_workflow_step:
|
|
49
|
+
conn.execute(text("DROP TABLE IF EXISTS workflow_steps"))
|
|
50
|
+
if old_workflow_run:
|
|
51
|
+
conn.execute(text("DROP TABLE IF EXISTS workflow_runs"))
|
|
52
|
+
|
|
53
|
+
# Recreate new schema
|
|
54
|
+
db.create_all() # creates workflow_runs, workflow_phases, workflow_steps
|
|
55
|
+
|
|
20
56
|
|
|
21
57
|
def create_app(config_class=None):
|
|
22
58
|
"""
|
|
@@ -45,7 +81,8 @@ def create_app(config_class=None):
|
|
|
45
81
|
|
|
46
82
|
# Create database tables
|
|
47
83
|
with app.app_context():
|
|
48
|
-
db.create_all()
|
|
84
|
+
# db.create_all()
|
|
85
|
+
reset_old_schema(db.engine, app.config['OUTPUT_FOLDER'])
|
|
49
86
|
|
|
50
87
|
# Additional setup
|
|
51
88
|
utils.create_gui_dir(app.config['OUTPUT_FOLDER'])
|
ivoryos/routes/api/api.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import copy
|
|
1
2
|
import os
|
|
2
3
|
from flask import Blueprint, jsonify, request, current_app
|
|
3
4
|
|
|
@@ -46,7 +47,7 @@ def backend_control(instrument: str=None):
|
|
|
46
47
|
current_app=current_app._get_current_object())
|
|
47
48
|
return jsonify(output), 200
|
|
48
49
|
|
|
49
|
-
snapshot = global_config.deck_snapshot
|
|
50
|
+
snapshot = copy.deepcopy(global_config.deck_snapshot)
|
|
50
51
|
# Iterate through each instrument in the snapshot
|
|
51
52
|
for instrument_key, instrument_data in snapshot.items():
|
|
52
53
|
# Iterate through each function associated with the current instrument
|
|
@@ -107,6 +107,15 @@
|
|
|
107
107
|
{{ field(class="btn btn-dark") }}
|
|
108
108
|
{% elif field.type == "BooleanField" %}
|
|
109
109
|
{{ field(class="form-check-input") }}
|
|
110
|
+
{% elif field.type == "FlexibleEnumField" %}
|
|
111
|
+
<input type="text" id="{{ field.id }}" name="{{ field.name }}" value="{{ field.data }}"
|
|
112
|
+
list="{{ field.id }}_options" placeholder="{{ field.render_kw.placeholder if field.render_kw and field.render_kw.placeholder }}"
|
|
113
|
+
class="form-control">
|
|
114
|
+
<datalist id="{{ field.id }}_options">
|
|
115
|
+
{% for key in field.choices %}
|
|
116
|
+
<option value="{{ key }}">{{ key }}</option>
|
|
117
|
+
{% endfor %}
|
|
118
|
+
</datalist>
|
|
110
119
|
{% else %}
|
|
111
120
|
{{ field(class="form-control") }}
|
|
112
121
|
{% endif %}
|
ivoryos/routes/data/data.py
CHANGED
|
@@ -3,7 +3,7 @@ import os
|
|
|
3
3
|
from flask import Blueprint, redirect, url_for, request, render_template, current_app, jsonify, send_file
|
|
4
4
|
from flask_login import login_required
|
|
5
5
|
|
|
6
|
-
from ivoryos.utils.db_models import db, WorkflowRun, WorkflowStep
|
|
6
|
+
from ivoryos.utils.db_models import db, WorkflowRun, WorkflowStep, WorkflowPhase
|
|
7
7
|
|
|
8
8
|
data = Blueprint('data', __name__, template_folder='templates')
|
|
9
9
|
|
|
@@ -37,7 +37,6 @@ def list_workflows():
|
|
|
37
37
|
else:
|
|
38
38
|
return render_template('workflow_database.html', workflows=workflows)
|
|
39
39
|
|
|
40
|
-
|
|
41
40
|
@data.get("/executions/records/<int:workflow_id>")
|
|
42
41
|
def workflow_logs(workflow_id:int):
|
|
43
42
|
"""
|
|
@@ -50,47 +49,87 @@ def workflow_logs(workflow_id:int):
|
|
|
50
49
|
:param workflow_id: workflow id
|
|
51
50
|
:type workflow_id: int
|
|
52
51
|
"""
|
|
52
|
+
workflow = db.session.get(WorkflowRun, workflow_id)
|
|
53
|
+
if not workflow:
|
|
54
|
+
return jsonify({"error": "Workflow not found"}), 404
|
|
55
|
+
|
|
56
|
+
# Query all phases for this run, ordered by start_time
|
|
57
|
+
phases = WorkflowPhase.query.filter_by(run_id=workflow_id).order_by(WorkflowPhase.start_time).all()
|
|
58
|
+
|
|
59
|
+
# Prepare grouped data for template (full objects)
|
|
60
|
+
grouped = {
|
|
61
|
+
"prep": [],
|
|
62
|
+
"script": {},
|
|
63
|
+
"cleanup": [],
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
# Prepare grouped data for JSON (dicts)
|
|
67
|
+
grouped_json = {
|
|
68
|
+
"prep": [],
|
|
69
|
+
"script": {},
|
|
70
|
+
"cleanup": [],
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
for phase in phases:
|
|
74
|
+
phase_dict = phase.as_dict()
|
|
75
|
+
|
|
76
|
+
# Steps sorted by step_index
|
|
77
|
+
steps = sorted(phase.steps, key=lambda s: s.step_index)
|
|
78
|
+
phase_steps_dicts = [s.as_dict() for s in steps]
|
|
79
|
+
|
|
80
|
+
if phase.name == "prep":
|
|
81
|
+
grouped["prep"].append(phase)
|
|
82
|
+
grouped_json["prep"].append({
|
|
83
|
+
**phase_dict,
|
|
84
|
+
"steps": phase_steps_dicts
|
|
85
|
+
})
|
|
53
86
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
grouped = {
|
|
60
|
-
"prep": [],
|
|
61
|
-
"script": {},
|
|
62
|
-
"cleanup": [],
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
# Use dicts for JSON response
|
|
66
|
-
grouped_json = {
|
|
67
|
-
"prep": [],
|
|
68
|
-
"script": {},
|
|
69
|
-
"cleanup": [],
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
for step in steps:
|
|
73
|
-
step_dict = step.as_dict()
|
|
74
|
-
|
|
75
|
-
if step.phase == "prep":
|
|
76
|
-
grouped["prep"].append(step)
|
|
77
|
-
grouped_json["prep"].append(step_dict)
|
|
78
|
-
|
|
79
|
-
elif step.phase == "script":
|
|
80
|
-
grouped["script"].setdefault(step.repeat_index, []).append(step)
|
|
81
|
-
grouped_json["script"].setdefault(step.repeat_index, []).append(step_dict)
|
|
82
|
-
|
|
83
|
-
elif step.phase == "cleanup" or step.method_name == "stop":
|
|
84
|
-
grouped["cleanup"].append(step)
|
|
85
|
-
grouped_json["cleanup"].append(step_dict)
|
|
86
|
-
|
|
87
|
-
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
88
|
-
return jsonify({
|
|
89
|
-
"workflow_info": workflow.as_dict(),
|
|
90
|
-
"steps": grouped_json,
|
|
87
|
+
elif phase.name == "main":
|
|
88
|
+
grouped["script"].setdefault(phase.repeat_index, []).append(phase)
|
|
89
|
+
grouped_json["script"].setdefault(phase.repeat_index, []).append({
|
|
90
|
+
**phase_dict,
|
|
91
|
+
"steps": phase_steps_dicts
|
|
91
92
|
})
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
|
|
94
|
+
elif phase.name == "cleanup":
|
|
95
|
+
grouped["cleanup"].append(phase)
|
|
96
|
+
grouped_json["cleanup"].append({
|
|
97
|
+
**phase_dict,
|
|
98
|
+
"steps": phase_steps_dicts
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
|
|
102
|
+
return jsonify({
|
|
103
|
+
"workflow_info": workflow.as_dict(),
|
|
104
|
+
"phases": grouped_json,
|
|
105
|
+
})
|
|
106
|
+
else:
|
|
107
|
+
return render_template("workflow_view.html", workflow=workflow, grouped=grouped)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@data.get("/executions/data/<int:workflow_id>")
|
|
111
|
+
def workflow_phase_data(workflow_id: int):
|
|
112
|
+
workflow = db.session.get(WorkflowRun, workflow_id)
|
|
113
|
+
if not workflow:
|
|
114
|
+
return jsonify({})
|
|
115
|
+
|
|
116
|
+
phase_data = {}
|
|
117
|
+
# Only plot 'main' phases
|
|
118
|
+
main_phases = WorkflowPhase.query.filter_by(run_id=workflow_id, name='main').order_by(
|
|
119
|
+
WorkflowPhase.repeat_index).all()
|
|
120
|
+
|
|
121
|
+
for phase in main_phases:
|
|
122
|
+
outputs = phase.outputs or {}
|
|
123
|
+
phase_index = phase.repeat_index
|
|
124
|
+
# Convert each key to list of dicts for x (phase_index) and y (value)
|
|
125
|
+
phase_data[phase_index] = {}
|
|
126
|
+
for k, v in outputs.items():
|
|
127
|
+
if isinstance(v, (int, float)):
|
|
128
|
+
phase_data[phase_index][k] = [{"x": phase_index, "y": v}]
|
|
129
|
+
elif isinstance(v, list) and all(isinstance(i, (int, float)) for i in v):
|
|
130
|
+
phase_data[phase_index][k] = v.map(lambda val, idx=0: {"x": phase_index, "y": val})
|
|
131
|
+
|
|
132
|
+
return jsonify(phase_data)
|
|
94
133
|
|
|
95
134
|
|
|
96
135
|
@data.delete("/executions/records/<int:workflow_id>")
|
|
@@ -1,13 +1,42 @@
|
|
|
1
|
-
<div class="card mb-2 {{ 'border-danger text-danger bg-light' if
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
</div>
|
|
1
|
+
<div class="card mb-2 {{ 'border-danger text-danger bg-light' if phase.run_error else 'border-secondary' }}">
|
|
2
|
+
<div class="card-body p-2">
|
|
3
|
+
<small class="text-muted">
|
|
4
|
+
<i class="fas fa-play-circle me-1"></i> Start: {{ phase.start_time.strftime('%H:%M:%S') if phase.start_time else 'N/A' }}
|
|
5
|
+
<i class="fas fa-stop-circle ms-2 me-1"></i> End: {{ phase.end_time.strftime('%H:%M:%S') if phase.end_time else 'N/A' }}
|
|
6
|
+
</small>
|
|
7
|
+
{% if phase.parameters %}
|
|
8
|
+
<div class="mt-2">
|
|
9
|
+
<strong>Parameters: </strong>
|
|
10
|
+
{% for key, value in phase.parameters.items() %}
|
|
11
|
+
<span class="badge bg-secondary me-1">{{ key }}: {{ value }}</span>
|
|
12
|
+
{% endfor %}
|
|
13
|
+
</div>
|
|
14
|
+
{% endif %}
|
|
15
|
+
{% if phase.steps %}
|
|
16
|
+
<div class="mt-2">
|
|
17
|
+
<strong>Steps:</strong>
|
|
18
|
+
<ul class="mb-0">
|
|
19
|
+
{% for step in phase.steps %}
|
|
20
|
+
<li class="{{ 'text-danger' if step.run_error else '' }}">
|
|
21
|
+
{{ step.method_name }}
|
|
22
|
+
<small class="text-muted">
|
|
23
|
+
({{ step.start_time.strftime('%H:%M:%S') if step.start_time else 'N/A' }} –
|
|
24
|
+
{{ step.end_time.strftime('%H:%M:%S') if step.end_time else 'N/A' }})
|
|
25
|
+
</small>
|
|
26
|
+
</li>
|
|
27
|
+
{% endfor %}
|
|
28
|
+
</ul>
|
|
29
|
+
</div>
|
|
30
|
+
{% endif %}
|
|
31
|
+
{% if phase.outputs %}
|
|
32
|
+
<div class="mt-1">
|
|
33
|
+
<strong>Outputs:</strong>
|
|
34
|
+
<ul class="mb-0">
|
|
35
|
+
{% for key, value in phase.outputs.items() %}
|
|
36
|
+
<li>{{ key }}: {{ value }}</li>
|
|
37
|
+
{% endfor %}
|
|
38
|
+
</ul>
|
|
39
|
+
</div>
|
|
40
|
+
{% endif %}
|
|
41
|
+
</div>
|
|
42
|
+
</div>
|
|
@@ -4,127 +4,348 @@
|
|
|
4
4
|
|
|
5
5
|
{% block body %}
|
|
6
6
|
<style>
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
7
|
+
.vis-time-axis .vis-text.vis-minor,
|
|
8
|
+
.vis-time-axis .vis-text.vis-major {
|
|
9
|
+
color: #666;
|
|
10
|
+
}
|
|
11
|
+
.vis-item.stop {
|
|
12
|
+
background-color: #dc3545;
|
|
13
|
+
color: white;
|
|
14
|
+
border: none;
|
|
15
|
+
font-weight: bold;
|
|
16
|
+
}
|
|
17
|
+
.vis-item.prep {
|
|
18
|
+
background-color: #17a2b8;
|
|
19
|
+
border-color: #138496;
|
|
20
|
+
}
|
|
21
|
+
.vis-item.script {
|
|
22
|
+
background-color: #28a745;
|
|
23
|
+
border-color: #1e7e34;
|
|
24
|
+
}
|
|
25
|
+
.vis-item.cleanup {
|
|
26
|
+
background-color: #ffc107;
|
|
27
|
+
border-color: #d39e00;
|
|
28
|
+
color: #212529;
|
|
29
|
+
}
|
|
30
|
+
#visualization {
|
|
31
|
+
border: 1px solid #dee2e6;
|
|
32
|
+
border-radius: 0.375rem;
|
|
33
|
+
background-color: #fff;
|
|
34
|
+
min-height: 200px;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
.section-header {
|
|
38
|
+
border-bottom: 2px solid #dee2e6;
|
|
39
|
+
padding-bottom: 0.5rem;
|
|
40
|
+
margin-bottom: 1rem;
|
|
41
|
+
color: #495057;
|
|
42
|
+
}
|
|
43
|
+
.loading-spinner {
|
|
44
|
+
display: none;
|
|
45
|
+
}
|
|
17
46
|
</style>
|
|
18
47
|
|
|
19
|
-
<div id="timeline"></div>
|
|
20
48
|
|
|
49
|
+
<div class="timeline-section" style="margin-bottom: 2rem;">
|
|
50
|
+
<h3 class="section-header">
|
|
51
|
+
<i class="fas fa-clock me-2"></i>Execution Timeline
|
|
52
|
+
</h3>
|
|
53
|
+
<div class="alert alert-info" role="alert">
|
|
54
|
+
<i class="fas fa-info-circle me-2"></i>
|
|
55
|
+
<strong>Tip:</strong> Click on timeline items to navigate to detailed views. Use Ctrl+scroll to zoom.
|
|
56
|
+
</div>
|
|
57
|
+
<div id="visualization"></div>
|
|
58
|
+
</div>
|
|
59
|
+
|
|
60
|
+
<!-- Phase Output Plot Section -->
|
|
61
|
+
|
|
62
|
+
<div class="data-section" style="margin-bottom: 2rem;">
|
|
63
|
+
<h3 class="section-header">
|
|
64
|
+
<div class="col-md-6">
|
|
65
|
+
<label for="output-select" class="form-label">Select Data Type:</label>
|
|
66
|
+
<select id="output-select" class="form-select">
|
|
67
|
+
<option value="">Loading data types...</option>
|
|
68
|
+
</select>
|
|
69
|
+
</div>
|
|
70
|
+
</h3>
|
|
71
|
+
|
|
72
|
+
<div class="plot-controls">
|
|
73
|
+
<div>
|
|
74
|
+
<div class="col-md-6 text-md-end">
|
|
75
|
+
<div class="loading-spinner">
|
|
76
|
+
<div class="spinner-border spinner-border-sm text-primary me-2" role="status">
|
|
77
|
+
<span class="visually-hidden">Loading...</span>
|
|
78
|
+
</div>
|
|
79
|
+
Loading plot data...
|
|
80
|
+
</div>
|
|
81
|
+
</div>
|
|
82
|
+
</div>
|
|
83
|
+
</div>
|
|
84
|
+
<div id="phase-plot" style="height:450px;" class="border rounded bg-white shadow-sm"></div>
|
|
85
|
+
</div>
|
|
86
|
+
|
|
87
|
+
<!-- Workflow Details Section -->
|
|
88
|
+
|
|
89
|
+
<div>
|
|
90
|
+
<h2 class="section-header">
|
|
91
|
+
<i class="fas fa-project-diagram me-2"></i>Workflow: {{ workflow.name }}
|
|
92
|
+
</h2>
|
|
93
|
+
|
|
94
|
+
<!-- Prep Phase -->
|
|
95
|
+
{% if grouped.prep %}
|
|
96
|
+
<div class="mb-4">
|
|
97
|
+
<h4 class="text-info mb-3">
|
|
98
|
+
<i class="fas fa-tools me-2"></i>Prep
|
|
99
|
+
</h4>
|
|
100
|
+
<div class="row">
|
|
101
|
+
{% for phase in grouped.prep %}
|
|
102
|
+
<div class="col-lg-6 mb-3">
|
|
103
|
+
{% include "components/step_card.html" %}
|
|
104
|
+
</div>
|
|
105
|
+
{% endfor %}
|
|
106
|
+
</div>
|
|
107
|
+
</div>
|
|
108
|
+
{% endif %}
|
|
109
|
+
|
|
110
|
+
<!-- Script Iterations -->
|
|
111
|
+
{% for repeat_index, phase_list in grouped.script.items()|sort %}
|
|
112
|
+
<div class="mb-4" id="card-iter{{ repeat_index }}">
|
|
113
|
+
<h4 class="text-success mb-3">
|
|
114
|
+
<i class="fas fa-redo me-2"></i>Iteration {{ repeat_index }}
|
|
115
|
+
</h4>
|
|
116
|
+
<div class="row">
|
|
117
|
+
{% for phase in phase_list %}
|
|
118
|
+
<div class="col-lg-6 mb-3">
|
|
119
|
+
{% include "components/step_card.html" %}
|
|
120
|
+
</div>
|
|
121
|
+
{% endfor %}
|
|
122
|
+
</div>
|
|
123
|
+
</div>
|
|
124
|
+
{% endfor %}
|
|
125
|
+
|
|
126
|
+
<!-- Cleanup Phase -->
|
|
127
|
+
{% if grouped.cleanup %}
|
|
128
|
+
<div>
|
|
129
|
+
<h4 class="text-warning mb-3">
|
|
130
|
+
<i class="fas fa-broom me-2"></i>Cleanup
|
|
131
|
+
</h4>
|
|
132
|
+
<div class="row">
|
|
133
|
+
{% for phase in grouped.cleanup %}
|
|
134
|
+
<div class="col-lg-6 mb-3">
|
|
135
|
+
{% include "components/step_card.html" %}
|
|
136
|
+
</div>
|
|
137
|
+
{% endfor %}
|
|
138
|
+
</div>
|
|
139
|
+
</div>
|
|
140
|
+
{% endif %}
|
|
141
|
+
</div>
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
<!-- External Dependencies -->
|
|
21
146
|
<script src="https://unpkg.com/vis-timeline@latest/standalone/umd/vis-timeline-graph2d.min.js"></script>
|
|
22
147
|
<link href="https://unpkg.com/vis-timeline@latest/styles/vis-timeline-graph2d.min.css" rel="stylesheet"/>
|
|
148
|
+
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
|
|
23
149
|
|
|
24
|
-
<
|
|
150
|
+
<script type="text/javascript">
|
|
151
|
+
document.addEventListener('DOMContentLoaded', function() {
|
|
152
|
+
// ---------------- Timeline Setup ----------------
|
|
153
|
+
const container = document.getElementById('visualization');
|
|
154
|
+
const groups = [
|
|
155
|
+
{ id: 'all', content: 'Workflow Execution' }
|
|
156
|
+
];
|
|
25
157
|
|
|
26
|
-
|
|
158
|
+
const items = [
|
|
159
|
+
{% if grouped.prep %}
|
|
160
|
+
{
|
|
161
|
+
id: 'prep',
|
|
162
|
+
content: 'Prep',
|
|
163
|
+
start: '{{ grouped.prep[0].start_time }}',
|
|
164
|
+
end: '{{ grouped.prep[-1].end_time }}',
|
|
165
|
+
className: 'prep',
|
|
166
|
+
group: 'all',
|
|
167
|
+
},
|
|
168
|
+
{% endif %}
|
|
27
169
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
170
|
+
{% for repeat_index, step_list in grouped.script.items()|sort %}
|
|
171
|
+
{
|
|
172
|
+
id: 'iter{{ repeat_index }}',
|
|
173
|
+
content: 'Iteration {{ repeat_index }}',
|
|
174
|
+
start: '{{ step_list[0].start_time }}',
|
|
175
|
+
end: '{{ step_list[-1].end_time }}',
|
|
176
|
+
className: 'script',
|
|
177
|
+
group: 'all',
|
|
178
|
+
},
|
|
179
|
+
{% for step in step_list %}
|
|
180
|
+
{% if step.method_name == "stop" %}
|
|
181
|
+
{
|
|
182
|
+
id: 'stop-{{ step.id }}',
|
|
183
|
+
content: '🛑 Stop',
|
|
184
|
+
start: '{{ step.start_time }}',
|
|
185
|
+
type: 'point',
|
|
186
|
+
className: 'stop',
|
|
187
|
+
group: 'all',
|
|
188
|
+
title: 'Stop event at {{ step.start_time }}'
|
|
189
|
+
},
|
|
190
|
+
{% endif %}
|
|
191
|
+
{% endfor %}
|
|
192
|
+
{% endfor %}
|
|
193
|
+
|
|
194
|
+
{% if grouped.cleanup %}
|
|
195
|
+
{
|
|
196
|
+
id: 'cleanup',
|
|
197
|
+
content: 'Cleanup ',
|
|
198
|
+
start: '{{ grouped.cleanup[0].start_time }}',
|
|
199
|
+
end: '{{ grouped.cleanup[-1].end_time }}',
|
|
200
|
+
className: 'cleanup',
|
|
201
|
+
group: 'all',
|
|
202
|
+
},
|
|
203
|
+
{% endif %}
|
|
204
|
+
];
|
|
205
|
+
|
|
206
|
+
var timeline = new vis.Timeline(container, items, groups, {
|
|
207
|
+
clickToUse: true,
|
|
208
|
+
stack: false, // keep items from overlapping vertically
|
|
209
|
+
horizontalScroll: true,
|
|
210
|
+
zoomKey: 'ctrlKey'
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
timeline.on('select', function (props) {
|
|
214
|
+
const id = props.items[0];
|
|
215
|
+
if (id && id.startsWith('iter')) {
|
|
216
|
+
const card = document.getElementById('card-' + id);
|
|
217
|
+
if (card) {
|
|
218
|
+
const yOffset = -80;
|
|
219
|
+
const y = card.getBoundingClientRect().top + window.pageYOffset + yOffset;
|
|
220
|
+
window.scrollTo({ top: y, behavior: 'smooth' });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
// ---------------- Phase Data Plot ----------------
|
|
226
|
+
const loadingSpinner = document.querySelector('.loading-spinner');
|
|
227
|
+
const select = document.getElementById('output-select');
|
|
228
|
+
|
|
229
|
+
loadingSpinner.style.display = 'block';
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
fetch("{{ url_for('data.workflow_phase_data', workflow_id=workflow.id) }}")
|
|
233
|
+
.then(res => {
|
|
234
|
+
if (!res.ok) {
|
|
235
|
+
throw new Error(`HTTP error! status: ${res.status}`);
|
|
236
|
+
}
|
|
237
|
+
return res.json();
|
|
238
|
+
})
|
|
239
|
+
.then(data => {
|
|
240
|
+
loadingSpinner.style.display = 'none';
|
|
241
|
+
|
|
242
|
+
const repeatKeys = Object.keys(data).sort((a, b) => a - b);
|
|
243
|
+
const dataSection = document.querySelector('.data-section'); // Get the entire section
|
|
244
|
+
|
|
245
|
+
if (!repeatKeys.length) {
|
|
246
|
+
// Hide the entire data section if no data
|
|
247
|
+
dataSection.style.display = 'none';
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const allKeys = new Set();
|
|
252
|
+
repeatKeys.forEach(k => {
|
|
253
|
+
Object.keys(data[k]).forEach(key => allKeys.add(key));
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
// If no keys found, also hide the section
|
|
257
|
+
if (allKeys.size === 0) {
|
|
258
|
+
dataSection.style.display = 'none';
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Show the data section since we have data
|
|
263
|
+
dataSection.style.display = 'block';
|
|
264
|
+
|
|
265
|
+
// Clear and populate select options
|
|
266
|
+
select.innerHTML = '';
|
|
267
|
+
allKeys.forEach(k => {
|
|
268
|
+
const option = new Option(k, k);
|
|
269
|
+
select.appendChild(option);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
function plotData(selectedKey) {
|
|
273
|
+
const x = [];
|
|
274
|
+
const y = [];
|
|
275
|
+
|
|
276
|
+
repeatKeys.forEach(repeat_index => {
|
|
277
|
+
const arr = data[repeat_index][selectedKey];
|
|
278
|
+
if (arr && arr.length) {
|
|
279
|
+
arr.forEach(d => {
|
|
280
|
+
if (typeof d === 'object' && d.x !== undefined && d.y !== undefined) {
|
|
281
|
+
x.push(d.x);
|
|
282
|
+
y.push(d.y);
|
|
283
|
+
} else if (typeof d === 'number') {
|
|
284
|
+
x.push(parseInt(repeat_index));
|
|
285
|
+
y.push(d);
|
|
286
|
+
}
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
const trace = {
|
|
292
|
+
x: x,
|
|
293
|
+
y: y,
|
|
294
|
+
mode: 'markers',
|
|
295
|
+
name: selectedKey,
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
const layout = {
|
|
299
|
+
xaxis: {
|
|
300
|
+
title: 'Iteration Index',
|
|
301
|
+
gridcolor: '#e9ecef'
|
|
302
|
+
},
|
|
303
|
+
yaxis: {
|
|
304
|
+
title: selectedKey,
|
|
305
|
+
gridcolor: '#e9ecef'
|
|
306
|
+
},
|
|
307
|
+
plot_bgcolor: '#ffffff',
|
|
308
|
+
paper_bgcolor: '#ffffff',
|
|
309
|
+
margin: { t: 60, r: 40, b: 60, l: 80 }
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
const config = {
|
|
313
|
+
responsive: true,
|
|
314
|
+
displayModeBar: true,
|
|
315
|
+
modeBarButtonsToRemove: ['lasso2d', 'select2d']
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
Plotly.newPlot('phase-plot', [trace], layout, config);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
select.addEventListener('change', e => {
|
|
322
|
+
if (e.target.value) {
|
|
323
|
+
plotData(e.target.value);
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
// Plot first available data type
|
|
328
|
+
if (allKeys.size > 0) {
|
|
329
|
+
plotData([...allKeys][0]);
|
|
330
|
+
}
|
|
331
|
+
})
|
|
332
|
+
.catch(error => {
|
|
333
|
+
loadingSpinner.style.display = 'none';
|
|
334
|
+
console.error('Error loading phase data:', error);
|
|
335
|
+
|
|
336
|
+
const dataSection = document.querySelector('.data-section');
|
|
337
|
+
// Hide the section on error as well
|
|
338
|
+
dataSection.style.display = 'none';
|
|
339
|
+
|
|
340
|
+
// Optionally, you could show an error message instead:
|
|
341
|
+
// dataSection.innerHTML = `
|
|
342
|
+
// <div class="alert alert-danger m-3" role="alert">
|
|
343
|
+
// <i class="fas fa-exclamation-triangle me-2"></i>
|
|
344
|
+
// <strong>Error:</strong> Unable to load phase data. ${error.message}
|
|
345
|
+
// </div>
|
|
346
|
+
// `;
|
|
347
|
+
});
|
|
348
|
+
});
|
|
106
349
|
</script>
|
|
107
350
|
|
|
108
|
-
<h2>Workflow: {{ workflow.name }}</h2>
|
|
109
|
-
|
|
110
|
-
{% if grouped.prep %}
|
|
111
|
-
<h4 class="mt-4">Prep Phase</h4>
|
|
112
|
-
{% for step in grouped.prep %}
|
|
113
|
-
{% include "components/step_card.html" %}
|
|
114
|
-
{% endfor %}
|
|
115
|
-
{% endif %}
|
|
116
|
-
|
|
117
|
-
{% for repeat_index, step_list in grouped.script.items()|sort %}
|
|
118
|
-
<h4 class="mt-4" id="card-iter{{ repeat_index }}">Iteration {{ repeat_index }}</h4>
|
|
119
|
-
{% for step in step_list %}
|
|
120
|
-
{% include "components/step_card.html" %}
|
|
121
|
-
{% endfor %}
|
|
122
|
-
{% endfor %}
|
|
123
|
-
|
|
124
|
-
{% if grouped.cleanup %}
|
|
125
|
-
<h4 class="mt-4">Cleanup Phase</h4>
|
|
126
|
-
{% for step in grouped.cleanup %}
|
|
127
|
-
{% include "components/step_card.html" %}
|
|
128
|
-
{% endfor %}
|
|
129
|
-
{% endif %}
|
|
130
351
|
{% endblock %}
|
ivoryos/server.py
CHANGED
|
@@ -94,12 +94,12 @@ def run(module=None, host="0.0.0.0", port=None, debug=None, llm_server=None, mod
|
|
|
94
94
|
app.config["MODULE"] = module
|
|
95
95
|
app.config["OFF_LINE"] = False
|
|
96
96
|
global_config.deck = sys.modules[module]
|
|
97
|
+
global_config.building_blocks = utils.create_block_snapshot()
|
|
97
98
|
global_config.deck_snapshot = utils.create_deck_snapshot(global_config.deck,
|
|
98
99
|
output_path=dummy_deck_path,
|
|
99
100
|
save=True,
|
|
100
101
|
exclude_names=exclude_names
|
|
101
102
|
)
|
|
102
|
-
global_config.building_blocks = utils.create_block_snapshot()
|
|
103
103
|
|
|
104
104
|
else:
|
|
105
105
|
app.config["OFF_LINE"] = True
|
ivoryos/utils/db_models.py
CHANGED
|
@@ -689,6 +689,7 @@ class Script(db.Model):
|
|
|
689
689
|
return "\n".join(lines)
|
|
690
690
|
|
|
691
691
|
class WorkflowRun(db.Model):
|
|
692
|
+
"""Represents the entire experiment"""
|
|
692
693
|
__tablename__ = 'workflow_runs'
|
|
693
694
|
|
|
694
695
|
id = db.Column(db.Integer, primary_key=True)
|
|
@@ -697,30 +698,65 @@ class WorkflowRun(db.Model):
|
|
|
697
698
|
start_time = db.Column(db.DateTime, default=datetime.now())
|
|
698
699
|
end_time = db.Column(db.DateTime)
|
|
699
700
|
data_path = db.Column(db.String(256))
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
701
|
+
repeat_mode = db.Column(db.String(64), default="none") # static_repeat, sweep, optimizer
|
|
702
|
+
|
|
703
|
+
# A run contains multiple iterations
|
|
704
|
+
phases = db.relationship(
|
|
705
|
+
'WorkflowPhase',
|
|
706
|
+
backref='workflow_runs', # Clearer back-reference name
|
|
703
707
|
cascade='all, delete-orphan',
|
|
704
|
-
|
|
708
|
+
lazy='dynamic' # Good for handling many iterations
|
|
705
709
|
)
|
|
706
710
|
def as_dict(self):
|
|
707
711
|
dict = self.__dict__
|
|
708
712
|
dict.pop('_sa_instance_state', None)
|
|
709
713
|
return dict
|
|
710
714
|
|
|
715
|
+
class WorkflowPhase(db.Model):
|
|
716
|
+
"""Represents a single function call within a WorkflowRun."""
|
|
717
|
+
__tablename__ = 'workflow_phases'
|
|
718
|
+
|
|
719
|
+
id = db.Column(db.Integer, primary_key=True)
|
|
720
|
+
# Foreign key to link this iteration to its parent run
|
|
721
|
+
run_id = db.Column(db.Integer, db.ForeignKey('workflow_runs.id', ondelete='CASCADE'), nullable=False)
|
|
722
|
+
|
|
723
|
+
# NEW: Store iteration-specific parameters here
|
|
724
|
+
name = db.Column(db.String(64), nullable=False) # 'prep', 'main', 'cleanup'
|
|
725
|
+
repeat_index = db.Column(db.Integer, default=0)
|
|
726
|
+
|
|
727
|
+
parameters = db.Column(JSONType) # Use db.JSON for general support
|
|
728
|
+
outputs = db.Column(JSONType)
|
|
729
|
+
start_time = db.Column(db.DateTime, default=datetime.now)
|
|
730
|
+
end_time = db.Column(db.DateTime)
|
|
731
|
+
|
|
732
|
+
# An iteration contains multiple steps
|
|
733
|
+
steps = db.relationship(
|
|
734
|
+
'WorkflowStep',
|
|
735
|
+
backref='workflow_phases', # Clearer back-reference name
|
|
736
|
+
cascade='all, delete-orphan'
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
def as_dict(self):
|
|
740
|
+
dict = self.__dict__.copy()
|
|
741
|
+
dict.pop('_sa_instance_state', None)
|
|
742
|
+
return dict
|
|
743
|
+
|
|
711
744
|
class WorkflowStep(db.Model):
|
|
712
745
|
__tablename__ = 'workflow_steps'
|
|
713
746
|
|
|
714
747
|
id = db.Column(db.Integer, primary_key=True)
|
|
715
|
-
workflow_id = db.Column(db.Integer, db.ForeignKey('workflow_runs.id', ondelete='CASCADE'), nullable=
|
|
748
|
+
# workflow_id = db.Column(db.Integer, db.ForeignKey('workflow_runs.id', ondelete='CASCADE'), nullable=True)
|
|
749
|
+
phase_id = db.Column(db.Integer, db.ForeignKey('workflow_phases.id', ondelete='CASCADE'), nullable=True)
|
|
716
750
|
|
|
717
|
-
phase = db.Column(db.String(64), nullable=False) # 'prep', 'main', 'cleanup'
|
|
718
|
-
repeat_index = db.Column(db.Integer, default=0) # Only applies to 'main' phase
|
|
751
|
+
# phase = db.Column(db.String(64), nullable=False) # 'prep', 'main', 'cleanup'
|
|
752
|
+
# repeat_index = db.Column(db.Integer, default=0) # Only applies to 'main' phase
|
|
719
753
|
step_index = db.Column(db.Integer, default=0)
|
|
720
754
|
method_name = db.Column(db.String(128), nullable=False)
|
|
721
755
|
start_time = db.Column(db.DateTime)
|
|
722
756
|
end_time = db.Column(db.DateTime)
|
|
723
757
|
run_error = db.Column(db.Boolean, default=False)
|
|
758
|
+
output = db.Column(JSONType, default={})
|
|
759
|
+
# Using as_dict method from ModelBase
|
|
724
760
|
|
|
725
761
|
def as_dict(self):
|
|
726
762
|
dict = self.__dict__.copy()
|
|
@@ -737,7 +773,7 @@ class SingleStep(db.Model):
|
|
|
737
773
|
start_time = db.Column(db.DateTime)
|
|
738
774
|
end_time = db.Column(db.DateTime)
|
|
739
775
|
run_error = db.Column(db.String(128))
|
|
740
|
-
output = db.Column(JSONType)
|
|
776
|
+
output = db.Column(JSONType, nullable=True)
|
|
741
777
|
|
|
742
778
|
def as_dict(self):
|
|
743
779
|
dict = self.__dict__.copy()
|
ivoryos/utils/form.py
CHANGED
|
@@ -210,10 +210,10 @@ class FlexibleEnumField(StringField):
|
|
|
210
210
|
if key in self.choices:
|
|
211
211
|
# Convert the string key to Enum instance
|
|
212
212
|
self.data = self.enum_class[key].value
|
|
213
|
-
elif
|
|
213
|
+
elif key.startswith("#"):
|
|
214
214
|
if not self.script.editing_type == "script":
|
|
215
215
|
raise ValueError(self.gettext("Variable is not supported in prep/cleanup"))
|
|
216
|
-
self.data =
|
|
216
|
+
self.data = key
|
|
217
217
|
else:
|
|
218
218
|
raise ValidationError(
|
|
219
219
|
f"Invalid choice: '{key}'. Must match one of {list(self.enum_class.__members__.keys())}")
|
|
@@ -286,7 +286,9 @@ def create_form_for_method(method, autofill, script=None, design=True):
|
|
|
286
286
|
# enum_class = [(e.name, e.value) for e in param.annotation]
|
|
287
287
|
field_class = FlexibleEnumField
|
|
288
288
|
placeholder_text = f"Choose or type a value for {param.annotation.__name__} (start with # for custom)"
|
|
289
|
+
|
|
289
290
|
extra_kwargs = {"choices": param.annotation}
|
|
291
|
+
|
|
290
292
|
else:
|
|
291
293
|
# print(param.annotation)
|
|
292
294
|
annotation, optional = parse_annotation(param.annotation)
|
ivoryos/utils/script_runner.py
CHANGED
|
@@ -6,7 +6,7 @@ import time
|
|
|
6
6
|
from datetime import datetime
|
|
7
7
|
|
|
8
8
|
from ivoryos.utils import utils, bo_campaign
|
|
9
|
-
from ivoryos.utils.db_models import Script, WorkflowRun, WorkflowStep, db,
|
|
9
|
+
from ivoryos.utils.db_models import Script, WorkflowRun, WorkflowStep, db, WorkflowPhase
|
|
10
10
|
from ivoryos.utils.global_config import GlobalConfig
|
|
11
11
|
from ivoryos.utils.decorators import BUILDING_BLOCKS
|
|
12
12
|
|
|
@@ -93,7 +93,7 @@ class ScriptRunner:
|
|
|
93
93
|
thread.start()
|
|
94
94
|
return thread
|
|
95
95
|
|
|
96
|
-
def exec_steps(self, script, section_name, logger, socketio,
|
|
96
|
+
def exec_steps(self, script, section_name, logger, socketio, phase_id, **kwargs):
|
|
97
97
|
"""
|
|
98
98
|
Executes a function defined in a string line by line
|
|
99
99
|
:param func_str: The function as a string
|
|
@@ -136,9 +136,9 @@ class ScriptRunner:
|
|
|
136
136
|
if self.stop_current_event.is_set():
|
|
137
137
|
logger.info(f'Stopping execution during {section_name}')
|
|
138
138
|
step = WorkflowStep(
|
|
139
|
-
|
|
140
|
-
phase=section_name,
|
|
141
|
-
repeat_index=i_progress,
|
|
139
|
+
phase_id=phase_id,
|
|
140
|
+
# phase=section_name,
|
|
141
|
+
# repeat_index=i_progress,
|
|
142
142
|
step_index=index,
|
|
143
143
|
method_name="stop",
|
|
144
144
|
start_time=datetime.now(),
|
|
@@ -148,21 +148,24 @@ class ScriptRunner:
|
|
|
148
148
|
db.session.add(step)
|
|
149
149
|
break
|
|
150
150
|
line = step_list[index]
|
|
151
|
-
|
|
152
|
-
|
|
151
|
+
|
|
152
|
+
method_name = line.strip()
|
|
153
|
+
# start_time = datetime.now()
|
|
154
|
+
|
|
153
155
|
step = WorkflowStep(
|
|
154
|
-
|
|
155
|
-
phase=section_name,
|
|
156
|
-
repeat_index=i_progress,
|
|
156
|
+
phase_id=phase_id,
|
|
157
|
+
# phase=section_name,
|
|
158
|
+
# repeat_index=i_progress,
|
|
157
159
|
step_index=index,
|
|
158
160
|
method_name=method_name,
|
|
159
|
-
start_time=
|
|
161
|
+
start_time=datetime.now(),
|
|
160
162
|
)
|
|
161
163
|
db.session.add(step)
|
|
162
164
|
db.session.commit()
|
|
165
|
+
|
|
163
166
|
logger.info(f"Executing: {line}")
|
|
164
167
|
socketio.emit('execution', {'section': f"{section_name}-{index}"})
|
|
165
|
-
|
|
168
|
+
|
|
166
169
|
# if line.startswith("registered_workflows"):
|
|
167
170
|
# line = line.replace("registered_workflows.", "")
|
|
168
171
|
try:
|
|
@@ -172,12 +175,13 @@ class ScriptRunner:
|
|
|
172
175
|
self.safe_sleep(duration)
|
|
173
176
|
else:
|
|
174
177
|
exec(line, exec_globals, exec_locals)
|
|
175
|
-
step.run_error = False
|
|
178
|
+
# step.run_error = False
|
|
176
179
|
|
|
177
180
|
except HumanInterventionRequired as e:
|
|
178
181
|
logger.warning(f"Human intervention required: {e}")
|
|
179
182
|
socketio.emit('human_intervention', {'message': str(e)})
|
|
180
183
|
# Instead of auto-resume, explicitly stay paused until user action
|
|
184
|
+
# step.run_error = False
|
|
181
185
|
self.toggle_pause()
|
|
182
186
|
|
|
183
187
|
except Exception as e:
|
|
@@ -187,7 +191,7 @@ class ScriptRunner:
|
|
|
187
191
|
step.run_error = True
|
|
188
192
|
self.toggle_pause()
|
|
189
193
|
step.end_time = datetime.now()
|
|
190
|
-
|
|
194
|
+
step.output = exec_locals
|
|
191
195
|
db.session.commit()
|
|
192
196
|
|
|
193
197
|
self.pause_event.wait()
|
|
@@ -195,10 +199,9 @@ class ScriptRunner:
|
|
|
195
199
|
# todo update script during the run
|
|
196
200
|
# _func_str = script.compile()
|
|
197
201
|
# step_list: list = script.convert_to_lines(_func_str).get(section_name, [])
|
|
198
|
-
if not step.run_error:
|
|
199
|
-
index += 1
|
|
200
|
-
elif not self.retry:
|
|
202
|
+
if not step.run_error or not self.retry:
|
|
201
203
|
index += 1
|
|
204
|
+
|
|
202
205
|
return exec_locals # Return the 'results' variable
|
|
203
206
|
|
|
204
207
|
def _run_with_stop_check(self, script: Script, repeat_count: int, run_name: str, logger, socketio, config, bo_args,
|
|
@@ -210,14 +213,17 @@ class ScriptRunner:
|
|
|
210
213
|
filename = None
|
|
211
214
|
error_flag = False
|
|
212
215
|
# create a new run entry in the database
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
216
|
+
repeat_mode = "batch" if config else "optimizer" if bo_args or optimizer else "repeat"
|
|
217
|
+
with current_app.app_context():
|
|
218
|
+
run = WorkflowRun(name=script.name or "untitled", platform=script.deck or "deck", start_time=datetime.now(),
|
|
219
|
+
repeat_mode=repeat_mode
|
|
220
|
+
)
|
|
221
|
+
db.session.add(run)
|
|
222
|
+
db.session.commit()
|
|
223
|
+
run_id = run.id # Save the ID
|
|
224
|
+
try:
|
|
220
225
|
|
|
226
|
+
global_config.runner_status = {"id":run_id, "type": "workflow"}
|
|
221
227
|
# Run "prep" section once
|
|
222
228
|
self._run_actions(script, section_name="prep", logger=logger, socketio=socketio, run_id=run_id)
|
|
223
229
|
output_list = []
|
|
@@ -234,35 +240,52 @@ class ScriptRunner:
|
|
|
234
240
|
# Run "cleanup" section once
|
|
235
241
|
self._run_actions(script, section_name="cleanup", logger=logger, socketio=socketio,run_id=run_id)
|
|
236
242
|
# Reset the running flag when done
|
|
237
|
-
|
|
238
243
|
# Save results if necessary
|
|
239
|
-
|
|
240
244
|
if not script.python_script and output_list:
|
|
241
245
|
filename = self._save_results(run_name, arg_type, return_list, output_list, logger, output_path)
|
|
242
246
|
self._emit_progress(socketio, 100)
|
|
243
247
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
248
|
+
except Exception as e:
|
|
249
|
+
logger.error(f"Error during script execution: {e.__str__()}")
|
|
250
|
+
error_flag = True
|
|
251
|
+
finally:
|
|
252
|
+
self.lock.release()
|
|
253
|
+
with current_app.app_context():
|
|
254
|
+
run = db.session.get(WorkflowRun, run_id)
|
|
255
|
+
run.end_time = datetime.now()
|
|
256
|
+
run.data_path = filename
|
|
257
|
+
run.run_error = error_flag
|
|
258
|
+
db.session.commit()
|
|
255
259
|
|
|
256
260
|
|
|
257
261
|
def _run_actions(self, script, section_name="", logger=None, socketio=None, run_id=None):
|
|
258
262
|
_func_str = script.python_script or script.compile()
|
|
259
263
|
step_list: list = script.convert_to_lines(_func_str).get(section_name, [])
|
|
260
|
-
|
|
264
|
+
if not step_list:
|
|
265
|
+
logger.info(f'No {section_name} steps')
|
|
266
|
+
return None
|
|
267
|
+
|
|
268
|
+
logger.info(f'Executing {section_name} steps')
|
|
261
269
|
if self.stop_pending_event.is_set():
|
|
262
270
|
logger.info(f"Stopping execution during {section_name} section.")
|
|
263
|
-
return
|
|
264
|
-
|
|
265
|
-
|
|
271
|
+
return None
|
|
272
|
+
|
|
273
|
+
phase = WorkflowPhase(
|
|
274
|
+
run_id=run_id,
|
|
275
|
+
name=section_name,
|
|
276
|
+
repeat_index=0,
|
|
277
|
+
start_time=datetime.now()
|
|
278
|
+
)
|
|
279
|
+
db.session.add(phase)
|
|
280
|
+
db.session.commit()
|
|
281
|
+
phase_id = phase.id
|
|
282
|
+
|
|
283
|
+
step_outputs = self.exec_steps(script, section_name, logger, socketio, phase_id=phase_id)
|
|
284
|
+
# Save phase-level output
|
|
285
|
+
phase.outputs = step_outputs
|
|
286
|
+
phase.end_time = datetime.now()
|
|
287
|
+
db.session.commit()
|
|
288
|
+
return step_outputs
|
|
266
289
|
|
|
267
290
|
def _run_config_section(self, config, arg_type, output_list, script, run_name, logger, socketio, run_id, compiled=True):
|
|
268
291
|
if not compiled:
|
|
@@ -285,10 +308,25 @@ class ScriptRunner:
|
|
|
285
308
|
self._emit_progress(socketio, progress)
|
|
286
309
|
# fname = f"{run_name}_script"
|
|
287
310
|
# function = self.globals_dict[fname]
|
|
288
|
-
|
|
311
|
+
|
|
312
|
+
phase = WorkflowPhase(
|
|
313
|
+
run_id=run_id,
|
|
314
|
+
name="main",
|
|
315
|
+
repeat_index=i,
|
|
316
|
+
parameters=kwargs,
|
|
317
|
+
start_time=datetime.now()
|
|
318
|
+
)
|
|
319
|
+
db.session.add(phase)
|
|
320
|
+
db.session.commit()
|
|
321
|
+
|
|
322
|
+
phase_id = phase.id
|
|
323
|
+
output = self.exec_steps(script, "script", logger, socketio, phase_id, **kwargs)
|
|
289
324
|
if output:
|
|
290
325
|
# kwargs.update(output)
|
|
291
326
|
output_list.append(output)
|
|
327
|
+
phase.outputs = {k:v for k, v in output.items() if k not in arg_type.keys()}
|
|
328
|
+
phase.end_time = datetime.now()
|
|
329
|
+
db.session.commit()
|
|
292
330
|
|
|
293
331
|
def _run_repeat_section(self, repeat_count, arg_types, bo_args, output_list, script, run_name, return_list, compiled,
|
|
294
332
|
logger, socketio, history, output_path, run_id, optimizer=None):
|
|
@@ -325,6 +363,17 @@ class ScriptRunner:
|
|
|
325
363
|
if self.stop_pending_event.is_set():
|
|
326
364
|
logger.info(f'Stopping execution during {run_name}: {i_progress + 1}/{int(repeat_count)}')
|
|
327
365
|
break
|
|
366
|
+
|
|
367
|
+
phase = WorkflowPhase(
|
|
368
|
+
run_id=run_id,
|
|
369
|
+
name="main",
|
|
370
|
+
repeat_index=i_progress,
|
|
371
|
+
start_time=datetime.now()
|
|
372
|
+
)
|
|
373
|
+
db.session.add(phase)
|
|
374
|
+
db.session.commit()
|
|
375
|
+
phase_id = phase.id
|
|
376
|
+
|
|
328
377
|
logger.info(f'Executing {run_name} experiment: {i_progress + 1}/{int(repeat_count)}')
|
|
329
378
|
progress = (i_progress + 1) * 100 / int(repeat_count) - 0.1
|
|
330
379
|
self._emit_progress(socketio, progress)
|
|
@@ -334,7 +383,9 @@ class ScriptRunner:
|
|
|
334
383
|
logger.info(f'Output value: {parameters}')
|
|
335
384
|
# fname = f"{run_name}_script"
|
|
336
385
|
# function = self.globals_dict[fname]
|
|
337
|
-
|
|
386
|
+
phase.parameters = parameters
|
|
387
|
+
|
|
388
|
+
output = self.exec_steps(script, "script", logger, socketio, phase_id, **parameters)
|
|
338
389
|
|
|
339
390
|
_output = {key: value for key, value in output.items() if key in return_list}
|
|
340
391
|
ax_client.complete_trial(trial_index=trial_index, raw_data=_output)
|
|
@@ -346,7 +397,8 @@ class ScriptRunner:
|
|
|
346
397
|
try:
|
|
347
398
|
parameters = optimizer.suggest(1)
|
|
348
399
|
logger.info(f'Output value: {parameters}')
|
|
349
|
-
|
|
400
|
+
phase.parameters = parameters
|
|
401
|
+
output = self.exec_steps(script, "script", logger, socketio, phase_id, **parameters)
|
|
350
402
|
if output:
|
|
351
403
|
optimizer.observe(output)
|
|
352
404
|
output.update(parameters)
|
|
@@ -356,17 +408,20 @@ class ScriptRunner:
|
|
|
356
408
|
else:
|
|
357
409
|
# fname = f"{run_name}_script"
|
|
358
410
|
# function = self.globals_dict[fname]
|
|
359
|
-
output = self.exec_steps(script, "script", logger, socketio,
|
|
411
|
+
output = self.exec_steps(script, "script", logger, socketio, phase_id)
|
|
360
412
|
|
|
361
413
|
if output:
|
|
362
414
|
output_list.append(output)
|
|
363
415
|
logger.info(f'Output value: {output}')
|
|
416
|
+
phase.outputs = output
|
|
417
|
+
phase.end_time = datetime.now()
|
|
418
|
+
db.session.commit()
|
|
364
419
|
|
|
365
420
|
if bo_args:
|
|
366
421
|
ax_client.save_to_json_file(os.path.join(output_path, f"{run_name}_ax_client.json"))
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
422
|
+
logger.info(
|
|
423
|
+
f'Optimization complete. Results saved to {os.path.join(output_path, f"{run_name}_ax_client.json")}'
|
|
424
|
+
)
|
|
370
425
|
return output_list
|
|
371
426
|
|
|
372
427
|
@staticmethod
|
ivoryos/utils/utils.py
CHANGED
|
@@ -349,8 +349,8 @@ def create_deck_snapshot(deck, save: bool = False, output_path: str = '', exclud
|
|
|
349
349
|
for name, class_type in items.items():
|
|
350
350
|
print(f" {name}: {class_type}")
|
|
351
351
|
|
|
352
|
-
print_section("✅ INCLUDED", deck_summary["included"])
|
|
353
|
-
print_section("❌ FAILED", deck_summary["failed"])
|
|
352
|
+
print_section("✅ INCLUDED MODULES", deck_summary["included"])
|
|
353
|
+
print_section("❌ FAILED MODULES", deck_summary["failed"])
|
|
354
354
|
print("\n")
|
|
355
355
|
|
|
356
356
|
print_deck_snapshot(deck_summary)
|
|
@@ -380,7 +380,10 @@ def create_block_snapshot(save: bool = False, output_path: str = ''):
|
|
|
380
380
|
"docstring": meta["docstring"],
|
|
381
381
|
"path": f"{func.__module__}.{func.__qualname__}"
|
|
382
382
|
}
|
|
383
|
-
|
|
383
|
+
if block_snapshot:
|
|
384
|
+
print(f"\n=== ✅ BUILDING_BLOCKS ({len(block_snapshot)}) ===")
|
|
385
|
+
for category, blocks in block_snapshot.items():
|
|
386
|
+
print(f" {category}: ", ",".join(blocks.keys()))
|
|
384
387
|
return block_snapshot
|
|
385
388
|
|
|
386
389
|
def load_deck(pkl_name: str):
|
ivoryos/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "1.
|
|
1
|
+
__version__ = "1.3.0"
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
ivoryos/__init__.py,sha256=eUtNgSskl--l94VUTT1bgiBR8gdMMFQgjHEsHOxdHyI,320
|
|
2
|
-
ivoryos/app.py,sha256=
|
|
2
|
+
ivoryos/app.py,sha256=tnFimgKnjRLhgIiraAVhEZFmcp6TGho7mQ5a5Y35rlY,4794
|
|
3
3
|
ivoryos/config.py,sha256=y3RxNjiIola9tK7jg-mHM8EzLMwiLwOzoisXkDvj0gA,2174
|
|
4
|
-
ivoryos/server.py,sha256=
|
|
4
|
+
ivoryos/server.py,sha256=K0_Ioui0uKshKl5fxGB_1wJD4OckXyR9DdOfCIhvkfE,6742
|
|
5
5
|
ivoryos/socket_handlers.py,sha256=VWVWiIdm4jYAutwGu6R0t1nK5MuMyOCL0xAnFn06jWQ,1302
|
|
6
|
-
ivoryos/version.py,sha256=
|
|
6
|
+
ivoryos/version.py,sha256=F5mW07pSyGrqDNY2Ehr-UpDzpBtN-FsYU0QGZWf6PJE,22
|
|
7
7
|
ivoryos/optimizer/ax_optimizer.py,sha256=PoSu8hrDFFpqyhRBnaSMswIUsDfEX6sPWt8NEZ_sobs,7112
|
|
8
8
|
ivoryos/optimizer/base_optimizer.py,sha256=JTbUharZKn0t8_BDbAFuwZIbT1VOnX1Xuog1pJuU8hY,1992
|
|
9
9
|
ivoryos/optimizer/baybe_optimizer.py,sha256=EdrrRiYO-IOx610cPXiQhH4qG8knUP0uiZ0YoyaGIU8,7954
|
|
10
10
|
ivoryos/optimizer/registry.py,sha256=lr0cqdI2iEjw227ZPRpVkvsdYdddjeJJRzawDv77cEc,219
|
|
11
11
|
ivoryos/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
ivoryos/routes/api/api.py,sha256=
|
|
12
|
+
ivoryos/routes/api/api.py,sha256=97Y7pqTwOaWgZgI5ovEPxEBm6Asrt0Iy0VhBkVp2xqA,2304
|
|
13
13
|
ivoryos/routes/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
14
|
ivoryos/routes/auth/auth.py,sha256=CqoP9cM8BuXVGHGujX7-0sNAOdWILU9amyBrObOD6Ss,3283
|
|
15
15
|
ivoryos/routes/auth/templates/login.html,sha256=WSRrKbdM_oobqSXFRTo-j9UlOgp6sYzS9tm7TqqPULI,1207
|
|
@@ -19,13 +19,13 @@ ivoryos/routes/control/control.py,sha256=6LnVF4mGgfLQvzmrSFxaFz9lBtBe4WnXlIouDxt
|
|
|
19
19
|
ivoryos/routes/control/control_file.py,sha256=3fQ9R8EcdqKs_hABn2EqRAB1xC2DHAT_q_pwsMIDDQI,864
|
|
20
20
|
ivoryos/routes/control/control_new_device.py,sha256=mfJKg5JAOagIpUKbp2b5nRwvd2V3bzT3M0zIhIsEaFM,5456
|
|
21
21
|
ivoryos/routes/control/utils.py,sha256=XlhhqAtOj7n3XfHPDxJ8TvCV2K2I2IixB0CBkl1QeQc,1242
|
|
22
|
-
ivoryos/routes/control/templates/controllers.html,sha256=
|
|
22
|
+
ivoryos/routes/control/templates/controllers.html,sha256=5hF3zcx5Rpy0Zaoq-5YGrR_TvPD9MGIa30fI4smEii0,9702
|
|
23
23
|
ivoryos/routes/control/templates/controllers_new.html,sha256=eVeLABT39DWOIYrwWClw7sAD3lCoAGCznygPgFbQoRc,5945
|
|
24
24
|
ivoryos/routes/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
-
ivoryos/routes/data/data.py,sha256=
|
|
25
|
+
ivoryos/routes/data/data.py,sha256=wJCd9TytdYCeU6BaGEUhBQHRYo7yn9OagIWa2qwSZEo,5583
|
|
26
26
|
ivoryos/routes/data/templates/workflow_database.html,sha256=ofvHcovpwmJXo1SFiSrL8I9kLU_3U1UxsJUUrQ2CJUU,4878
|
|
27
|
-
ivoryos/routes/data/templates/workflow_view.html,sha256=
|
|
28
|
-
ivoryos/routes/data/templates/components/step_card.html,sha256=
|
|
27
|
+
ivoryos/routes/data/templates/workflow_view.html,sha256=Ti17kzlPlYTmzx5MkdsPlXJ1_k6QgMYQBM6FHjG50go,12491
|
|
28
|
+
ivoryos/routes/data/templates/components/step_card.html,sha256=XWsr7qxAY76RCuQHETubWjWBlPgs2HkviH4ju6qfBKo,1923
|
|
29
29
|
ivoryos/routes/design/__init__.py,sha256=zS3HXKaw0ALL5n6t_W1rUz5Uj5_tTQ-Y1VMXyzewvR0,113
|
|
30
30
|
ivoryos/routes/design/design.py,sha256=xYDwtCdTcCd282guaIeNvfUFc5UsiypkQVpRvFqRujQ,18246
|
|
31
31
|
ivoryos/routes/design/design_file.py,sha256=m4yku8fkpLUs4XvLJBqR5V-kyaGKbGB6ZoRxGbjEU5Q,2140
|
|
@@ -86,18 +86,18 @@ ivoryos/templates/base.html,sha256=cl5w6E8yskbUzdiJFal6fZjnPuFNKEzc7BrrbRd6bMI,8
|
|
|
86
86
|
ivoryos/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
87
87
|
ivoryos/utils/bo_campaign.py,sha256=Fil-zT7JexL_p9XqyWByjAk42XB1R9XUKN8CdV5bi6c,9714
|
|
88
88
|
ivoryos/utils/client_proxy.py,sha256=74G3HAuq50iEHkSvlMZFmQaukm613FbRgOdzO_T3dMg,10191
|
|
89
|
-
ivoryos/utils/db_models.py,sha256=
|
|
89
|
+
ivoryos/utils/db_models.py,sha256=TaRA65Zmj2lIqQk5sDoQzP_od6QnPUYJgvDAV9LkqYM,31074
|
|
90
90
|
ivoryos/utils/decorators.py,sha256=p1Bdl3dCeaHNv6-cCCUOZMiFu9kRaqqQnkFJUkzPoJE,991
|
|
91
|
-
ivoryos/utils/form.py,sha256=
|
|
91
|
+
ivoryos/utils/form.py,sha256=Ej9tx06KZZ5fPQm1ho1byotNocF3u24aatc2ZyI0rK4,22301
|
|
92
92
|
ivoryos/utils/global_config.py,sha256=D6oz5dttyaP24jbqnw1sR64moSb-7jJkSpRuufdA_TI,2747
|
|
93
93
|
ivoryos/utils/llm_agent.py,sha256=-lVCkjPlpLues9sNTmaT7bT4sdhWvV2DiojNwzB2Lcw,6422
|
|
94
94
|
ivoryos/utils/py_to_json.py,sha256=fyqjaxDHPh-sahgT6IHSn34ktwf6y51_x1qvhbNlH-U,7314
|
|
95
|
-
ivoryos/utils/script_runner.py,sha256=
|
|
95
|
+
ivoryos/utils/script_runner.py,sha256=TksUOaisnGj-A3lT78Ni6ilK_huc96WSq2uUPEcxdek,19354
|
|
96
96
|
ivoryos/utils/serilize.py,sha256=lkBhkz8r2bLmz2_xOb0c4ptSSOqjIu6krj5YYK4Nvj8,6784
|
|
97
97
|
ivoryos/utils/task_runner.py,sha256=bfG6GubdlzgD8rBwzD00aGB5LDFmb9hLFJIOMH8hVv4,3248
|
|
98
|
-
ivoryos/utils/utils.py,sha256=
|
|
99
|
-
ivoryos-1.
|
|
100
|
-
ivoryos-1.
|
|
101
|
-
ivoryos-1.
|
|
102
|
-
ivoryos-1.
|
|
103
|
-
ivoryos-1.
|
|
98
|
+
ivoryos/utils/utils.py,sha256=09VPNRaIoA-mp1TXLGC3BwM2tDaAJ36csvNtW19KsU0,14792
|
|
99
|
+
ivoryos-1.3.0.dist-info/licenses/LICENSE,sha256=p2c8S8i-8YqMpZCJnadLz1-ofxnRMILzz6NCMIypRag,1084
|
|
100
|
+
ivoryos-1.3.0.dist-info/METADATA,sha256=OanJGtMTbt4C6D3IFz-GprxXAME0PkEPEfBLliu4Jfw,7351
|
|
101
|
+
ivoryos-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
102
|
+
ivoryos-1.3.0.dist-info/top_level.txt,sha256=FRIWWdiEvRKqw-XfF_UK3XV0CrnNb6EmVbEgjaVazRM,8
|
|
103
|
+
ivoryos-1.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|