ivoryos 1.2.8__tar.gz → 1.3.0__tar.gz

{ivoryos-1.2.8 → ivoryos-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivoryos
-Version: 1.2.8
+Version: 1.3.0
 Summary: an open-source Python package enabling Self-Driving Labs (SDLs) interoperability
 Author-email: Ivory Zhang <ivoryzhang@chem.ubc.ca>
 License: MIT

{ivoryos-1.2.8 → ivoryos-1.3.0}/ivoryos/app.py

@@ -17,6 +17,42 @@ from ivoryos.routes.api.api import api
 from ivoryos.socket_handlers import socketio
 from ivoryos.routes.main.main import main
 from ivoryos.version import __version__ as ivoryos_version
+from sqlalchemy import inspect, text
+from flask import current_app
+
+
+def reset_old_schema(engine, db_dir):
+    inspector = inspect(engine)
+    tables = inspector.get_table_names()
+
+
+    # Check if old tables exist (no workflow_phases table)
+    has_workflow_phase = 'workflow_phases' in tables
+    old_workflow_run = 'old_workflow_run' in tables
+    old_workflow_step = 'workflow_steps' in tables
+
+    if not has_workflow_phase:
+        print("⚠️ Old workflow database detected! All previous workflows have been reset to support the new schema.")
+        # Backup old DB
+        db_path = os.path.join(db_dir, "ivoryos.db")
+        if os.path.exists(db_path):
+            # os.makedirs(backup_dir, exist_ok=True)
+            from datetime import datetime
+            import shutil
+            ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+            backup_path = os.path.join(db_dir, f"ivoryos_backup_{ts}.db")
+            shutil.copy(db_path, backup_path)
+            print(f"Backup created at {backup_path}")
+        with engine.begin() as conn:
+            # Drop old tables
+            if old_workflow_step:
+                conn.execute(text("DROP TABLE IF EXISTS workflow_steps"))
+            if old_workflow_run:
+                conn.execute(text("DROP TABLE IF EXISTS workflow_runs"))
+
+    # Recreate new schema
+    db.create_all()  # creates workflow_runs, workflow_phases, workflow_steps
+
 
 def create_app(config_class=None):
     """
@@ -45,7 +81,8 @@ def create_app(config_class=None):
 
     # Create database tables
     with app.app_context():
-        db.create_all()
+        # db.create_all()
+        reset_old_schema(db.engine, app.config['OUTPUT_FOLDER'])
 
     # Additional setup
     utils.create_gui_dir(app.config['OUTPUT_FOLDER'])

{ivoryos-1.2.8 → ivoryos-1.3.0}/ivoryos/routes/api/api.py

@@ -1,3 +1,4 @@
+import copy
 import os
 from flask import Blueprint, jsonify, request, current_app
 
@@ -46,7 +47,7 @@ def backend_control(instrument: str=None):
                                             current_app=current_app._get_current_object())
             return jsonify(output), 200
 
-    snapshot = global_config.deck_snapshot.copy()
+    snapshot = copy.deepcopy(global_config.deck_snapshot)
     # Iterate through each instrument in the snapshot
     for instrument_key, instrument_data in snapshot.items():
         # Iterate through each function associated with the current instrument

{ivoryos-1.2.8 → ivoryos-1.3.0}/ivoryos/routes/control/templates/controllers.html

@@ -107,6 +107,15 @@
                                 {{ field(class="btn btn-dark") }}
                               {% elif field.type == "BooleanField" %}
                                 {{ field(class="form-check-input") }}
+                              {% elif field.type == "FlexibleEnumField" %}
+                                 <input type="text" id="{{ field.id }}" name="{{ field.name }}" value="{{ field.data }}"
+                                    list="{{ field.id }}_options" placeholder="{{ field.render_kw.placeholder if field.render_kw and field.render_kw.placeholder }}"
+                                    class="form-control">
+                                 <datalist id="{{ field.id }}_options">
+                                    {% for key in field.choices %}
+                                       <option value="{{ key }}">{{ key }}</option>
+                                    {% endfor %}
+                                 </datalist>
                               {% else %}
                                 {{ field(class="form-control") }}
                               {% endif %}

ivoryos-1.3.0/ivoryos/routes/data/data.py

@@ -0,0 +1,170 @@
+import os
+
+from flask import Blueprint, redirect, url_for, request, render_template, current_app, jsonify, send_file
+from flask_login import login_required
+
+from ivoryos.utils.db_models import db, WorkflowRun, WorkflowStep, WorkflowPhase
+
+data = Blueprint('data', __name__, template_folder='templates')
+
+
+
+@data.route('/executions/records')
+@login_required
+def list_workflows():
+    """
+    .. :quickref: Workflow Execution Database; list all workflow execution records
+
+    list all workflow execution records
+
+    .. http:get:: /executions/records
+
+    """
+    query = WorkflowRun.query.order_by(WorkflowRun.id.desc())
+    search_term = request.args.get("keyword", None)
+    if search_term:
+        query = query.filter(WorkflowRun.name.like(f'%{search_term}%'))
+    page = request.args.get('page', default=1, type=int)
+    per_page = 10
+
+    workflows = query.paginate(page=page, per_page=per_page, error_out=False)
+    if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
+        workflows = query.all()
+        workflow_data = {w.id:{"workflow_name":w.name, "start_time":w.start_time} for w in workflows}
+        return jsonify({
+            "workflow_data": workflow_data,
+        })
+    else:
+        return render_template('workflow_database.html', workflows=workflows)
+
+@data.get("/executions/records/<int:workflow_id>")
+def workflow_logs(workflow_id:int):
+    """
+    .. :quickref: Workflow Data Database; get workflow data, steps, and logs
+
+    get workflow data logs by workflow id
+
+    .. http:get:: /executions/<int:workflow_id>
+
+    :param workflow_id: workflow id
+    :type workflow_id: int
+    """
+    workflow = db.session.get(WorkflowRun, workflow_id)
+    if not workflow:
+        return jsonify({"error": "Workflow not found"}), 404
+
+    # Query all phases for this run, ordered by start_time
+    phases = WorkflowPhase.query.filter_by(run_id=workflow_id).order_by(WorkflowPhase.start_time).all()
+
+    # Prepare grouped data for template (full objects)
+    grouped = {
+        "prep": [],
+        "script": {},
+        "cleanup": [],
+    }
+
+    # Prepare grouped data for JSON (dicts)
+    grouped_json = {
+        "prep": [],
+        "script": {},
+        "cleanup": [],
+    }
+
+    for phase in phases:
+        phase_dict = phase.as_dict()
+
+        # Steps sorted by step_index
+        steps = sorted(phase.steps, key=lambda s: s.step_index)
+        phase_steps_dicts = [s.as_dict() for s in steps]
+
+        if phase.name == "prep":
+            grouped["prep"].append(phase)
+            grouped_json["prep"].append({
+                **phase_dict,
+                "steps": phase_steps_dicts
+            })
+
+        elif phase.name == "main":
+            grouped["script"].setdefault(phase.repeat_index, []).append(phase)
+            grouped_json["script"].setdefault(phase.repeat_index, []).append({
+                **phase_dict,
+                "steps": phase_steps_dicts
+            })
+
+        elif phase.name == "cleanup":
+            grouped["cleanup"].append(phase)
+            grouped_json["cleanup"].append({
+                **phase_dict,
+                "steps": phase_steps_dicts
+            })
+
+    if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
+        return jsonify({
+            "workflow_info": workflow.as_dict(),
+            "phases": grouped_json,
+        })
+    else:
+        return render_template("workflow_view.html", workflow=workflow, grouped=grouped)
+
+
+@data.get("/executions/data/<int:workflow_id>")
+def workflow_phase_data(workflow_id: int):
+    workflow = db.session.get(WorkflowRun, workflow_id)
+    if not workflow:
+        return jsonify({})
+
+    phase_data = {}
+    # Only plot 'main' phases
+    main_phases = WorkflowPhase.query.filter_by(run_id=workflow_id, name='main').order_by(
+        WorkflowPhase.repeat_index).all()
+
+    for phase in main_phases:
+        outputs = phase.outputs or {}
+        phase_index = phase.repeat_index
+        # Convert each key to list of dicts for x (phase_index) and y (value)
+        phase_data[phase_index] = {}
+        for k, v in outputs.items():
+            if isinstance(v, (int, float)):
+                phase_data[phase_index][k] = [{"x": phase_index, "y": v}]
+            elif isinstance(v, list) and all(isinstance(i, (int, float)) for i in v):
+                phase_data[phase_index][k] = v.map(lambda val, idx=0: {"x": phase_index, "y": val})
+
+    return jsonify(phase_data)
+
+
+@data.delete("/executions/records/<int:workflow_id>")
+@login_required
+def delete_workflow_record(workflow_id: int):
+    """
+    .. :quickref: Workflow Data Database; delete a workflow execution record
+
+    delete a workflow execution record by workflow id
+
+    .. http:delete:: /executions/records/<int:workflow_id>
+
+    :param workflow_id: workflow id
+    :type workflow_id: int
+    :status 200: return success message
+    """
+    run = WorkflowRun.query.get(workflow_id)
+    db.session.delete(run)
+    db.session.commit()
+    return jsonify(success=True)
+
+
+@data.route('/files/execution-data/<string:filename>')
+@login_required
+def download_results(filename:str):
+    """
+    .. :quickref: Workflow data; download a workflow data file (.CSV)
+
+    .. http:get:: /files/execution-data/<string:filename>
+
+    :param filename: workflow data filename
+    :type filename: str
+
+    # :status 302: load pseudo deck and then redirects to :http:get:`/ivoryos/executions`
+    """
+
+    filepath = os.path.join(current_app.config["DATA_FOLDER"], filename)
+    return send_file(os.path.abspath(filepath), as_attachment=True)

ivoryos-1.3.0/ivoryos/routes/data/templates/components/step_card.html

@@ -0,0 +1,42 @@
+<div class="card mb-2 {{ 'border-danger text-danger bg-light' if phase.run_error else 'border-secondary' }}">
+    <div class="card-body p-2">
+        <small class="text-muted">
+            <i class="fas fa-play-circle me-1"></i> Start: {{ phase.start_time.strftime('%H:%M:%S') if phase.start_time else 'N/A' }}
+            <i class="fas fa-stop-circle ms-2 me-1"></i> End: {{ phase.end_time.strftime('%H:%M:%S') if phase.end_time else 'N/A' }}
+        </small>
+            {% if phase.parameters %}
+            <div class="mt-2">
+                <strong>Parameters: </strong>
+                    {% for key, value in phase.parameters.items() %}
+                    <span class="badge bg-secondary me-1">{{ key }}: {{ value }}</span>
+                    {% endfor %}
+            </div>
+            {% endif %}
+        {% if phase.steps %}
+            <div class="mt-2">
+                <strong>Steps:</strong>
+                <ul class="mb-0">
+                    {% for step in phase.steps %}
+                        <li class="{{ 'text-danger' if step.run_error else '' }}">
+                            {{ step.method_name }}
+                            <small class="text-muted">
+                              ({{ step.start_time.strftime('%H:%M:%S') if step.start_time else 'N/A' }} –
+                              {{ step.end_time.strftime('%H:%M:%S') if step.end_time else 'N/A' }})
+                            </small>
+                        </li>
+                    {% endfor %}
+                </ul>
+            </div>
+        {% endif %}
+        {% if phase.outputs %}
+            <div class="mt-1">
+                <strong>Outputs:</strong>
+                <ul class="mb-0">
+                    {% for key, value in phase.outputs.items() %}
+                        <li>{{ key }}: {{ value }}</li>
+                    {% endfor %}
+                </ul>
+            </div>
+        {% endif %}
+    </div>
+</div>

ivoryos-1.3.0/ivoryos/routes/data/templates/workflow_view.html

@@ -0,0 +1,351 @@
+{% extends 'base.html' %}
+
+{% block title %}IvoryOS | Experiment Results{% endblock %}
+
+{% block body %}
+<style>
+    .vis-time-axis .vis-text.vis-minor,
+    .vis-time-axis .vis-text.vis-major {
+        color: #666;
+    }
+    .vis-item.stop {
+        background-color: #dc3545;
+        color: white;
+        border: none;
+        font-weight: bold;
+    }
+    .vis-item.prep {
+        background-color: #17a2b8;
+        border-color: #138496;
+    }
+    .vis-item.script {
+        background-color: #28a745;
+        border-color: #1e7e34;
+    }
+    .vis-item.cleanup {
+        background-color: #ffc107;
+        border-color: #d39e00;
+        color: #212529;
+    }
+    #visualization {
+        border: 1px solid #dee2e6;
+        border-radius: 0.375rem;
+        background-color: #fff;
+        min-height: 200px;
+    }
+
+    .section-header {
+        border-bottom: 2px solid #dee2e6;
+        padding-bottom: 0.5rem;
+        margin-bottom: 1rem;
+        color: #495057;
+    }
+    .loading-spinner {
+        display: none;
+    }
+</style>
+
+
+    <div class="timeline-section" style="margin-bottom: 2rem;">
+        <h3 class="section-header">
+            <i class="fas fa-clock me-2"></i>Execution Timeline
+        </h3>
+        <div class="alert alert-info" role="alert">
+            <i class="fas fa-info-circle me-2"></i>
+            <strong>Tip:</strong> Click on timeline items to navigate to detailed views. Use Ctrl+scroll to zoom.
+        </div>
+        <div id="visualization"></div>
+    </div>
+
+    <!-- Phase Output Plot Section -->
+
+    <div class="data-section" style="margin-bottom: 2rem;">
+        <h3 class="section-header">
+            <div class="col-md-6">
+                <label for="output-select" class="form-label">Select Data Type:</label>
+                <select id="output-select" class="form-select">
+                    <option value="">Loading data types...</option>
+                </select>
+            </div>
+        </h3>
+
+        <div class="plot-controls">
+            <div>
+                <div class="col-md-6 text-md-end">
+                    <div class="loading-spinner">
+                        <div class="spinner-border spinner-border-sm text-primary me-2" role="status">
+                            <span class="visually-hidden">Loading...</span>
+                        </div>
+                        Loading plot data...
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div id="phase-plot" style="height:450px;" class="border rounded bg-white shadow-sm"></div>
+    </div>
+
+    <!-- Workflow Details Section -->
+
+    <div>
+        <h2 class="section-header">
+            <i class="fas fa-project-diagram me-2"></i>Workflow: {{ workflow.name }}
+        </h2>
+
+        <!-- Prep Phase -->
+        {% if grouped.prep %}
+            <div class="mb-4">
+                <h4 class="text-info mb-3">
+                    <i class="fas fa-tools me-2"></i>Prep
+                </h4>
+                <div class="row">
+                    {% for phase in grouped.prep %}
+                        <div class="col-lg-6 mb-3">
+                            {% include "components/step_card.html" %}
+                        </div>
+                    {% endfor %}
+                </div>
+            </div>
+        {% endif %}
+
+        <!-- Script Iterations -->
+        {% for repeat_index, phase_list in grouped.script.items()|sort %}
+            <div class="mb-4" id="card-iter{{ repeat_index }}">
+                <h4 class="text-success mb-3">
+                    <i class="fas fa-redo me-2"></i>Iteration {{ repeat_index }}
+                </h4>
+                <div class="row">
+                    {% for phase in phase_list %}
+                        <div class="col-lg-6 mb-3">
+                            {% include "components/step_card.html" %}
+                        </div>
+                    {% endfor %}
+                </div>
+            </div>
+        {% endfor %}
+
+        <!-- Cleanup Phase -->
+        {% if grouped.cleanup %}
+            <div>
+                <h4 class="text-warning mb-3">
+                    <i class="fas fa-broom me-2"></i>Cleanup
+                </h4>
+                <div class="row">
+                    {% for phase in grouped.cleanup %}
+                        <div class="col-lg-6 mb-3">
+                            {% include "components/step_card.html" %}
+                        </div>
+                    {% endfor %}
+                </div>
+            </div>
+        {% endif %}
+    </div>
+
+
+
+<!-- External Dependencies -->
+<script src="https://unpkg.com/vis-timeline@latest/standalone/umd/vis-timeline-graph2d.min.js"></script>
+<link href="https://unpkg.com/vis-timeline@latest/styles/vis-timeline-graph2d.min.css" rel="stylesheet"/>
+<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+
+<script type="text/javascript">
+    document.addEventListener('DOMContentLoaded', function() {
+        // ---------------- Timeline Setup ----------------
+        const container = document.getElementById('visualization');
+        const groups = [
+            { id: 'all', content: 'Workflow Execution' }
+        ];
+
+        const items = [
+            {% if grouped.prep %}
+                {
+                    id: 'prep',
+                    content: 'Prep',
+                    start: '{{ grouped.prep[0].start_time }}',
+                    end: '{{ grouped.prep[-1].end_time }}',
+                    className: 'prep',
+                    group: 'all',
+                },
+            {% endif %}
+
+            {% for repeat_index, step_list in grouped.script.items()|sort %}
+                {
+                    id: 'iter{{ repeat_index }}',
+                    content: 'Iteration {{ repeat_index }}',
+                    start: '{{ step_list[0].start_time }}',
+                    end: '{{ step_list[-1].end_time }}',
+                    className: 'script',
+                    group: 'all',
+                },
+                {% for step in step_list %}
+                    {% if step.method_name == "stop" %}
+                        {
+                            id: 'stop-{{ step.id }}',
+                            content: '🛑 Stop',
+                            start: '{{ step.start_time }}',
+                            type: 'point',
+                            className: 'stop',
+                            group: 'all',
+                            title: 'Stop event at {{ step.start_time }}'
+                        },
+                    {% endif %}
+                {% endfor %}
+            {% endfor %}
+
+            {% if grouped.cleanup %}
+                {
+                    id: 'cleanup',
+                    content: 'Cleanup ',
+                    start: '{{ grouped.cleanup[0].start_time }}',
+                    end: '{{ grouped.cleanup[-1].end_time }}',
+                    className: 'cleanup',
+                    group: 'all',
+                },
+            {% endif %}
+        ];
+
+        var timeline = new vis.Timeline(container, items, groups, {
+            clickToUse: true,
+            stack: false,        // keep items from overlapping vertically
+            horizontalScroll: true,
+            zoomKey: 'ctrlKey'
+        });
+
+        timeline.on('select', function (props) {
+            const id = props.items[0];
+            if (id && id.startsWith('iter')) {
+                const card = document.getElementById('card-' + id);
+                if (card) {
+                    const yOffset = -80;
+                    const y = card.getBoundingClientRect().top + window.pageYOffset + yOffset;
+                    window.scrollTo({ top: y, behavior: 'smooth' });
+                }
+            }
+        });
+
+        // ---------------- Phase Data Plot ----------------
+        const loadingSpinner = document.querySelector('.loading-spinner');
+        const select = document.getElementById('output-select');
+
+        loadingSpinner.style.display = 'block';
+
+
+        fetch("{{ url_for('data.workflow_phase_data', workflow_id=workflow.id) }}")
+            .then(res => {
+                if (!res.ok) {
+                    throw new Error(`HTTP error! status: ${res.status}`);
+                }
+                return res.json();
+            })
+            .then(data => {
+                loadingSpinner.style.display = 'none';
+
+                const repeatKeys = Object.keys(data).sort((a, b) => a - b);
+                const dataSection = document.querySelector('.data-section'); // Get the entire section
+
+                if (!repeatKeys.length) {
+                    // Hide the entire data section if no data
+                    dataSection.style.display = 'none';
+                    return;
+                }
+
+                const allKeys = new Set();
+                repeatKeys.forEach(k => {
+                    Object.keys(data[k]).forEach(key => allKeys.add(key));
+                });
+
+                // If no keys found, also hide the section
+                if (allKeys.size === 0) {
+                    dataSection.style.display = 'none';
+                    return;
+                }
+
+                // Show the data section since we have data
+                dataSection.style.display = 'block';
+
+                // Clear and populate select options
+                select.innerHTML = '';
+                allKeys.forEach(k => {
+                    const option = new Option(k, k);
+                    select.appendChild(option);
+                });
+
+                function plotData(selectedKey) {
+                    const x = [];
+                    const y = [];
+
+                    repeatKeys.forEach(repeat_index => {
+                        const arr = data[repeat_index][selectedKey];
+                        if (arr && arr.length) {
+                            arr.forEach(d => {
+                                if (typeof d === 'object' && d.x !== undefined && d.y !== undefined) {
+                                    x.push(d.x);
+                                    y.push(d.y);
+                                } else if (typeof d === 'number') {
+                                    x.push(parseInt(repeat_index));
+                                    y.push(d);
+                                }
+                            });
+                        }
+                    });
+
+                    const trace = {
+                        x: x,
+                        y: y,
+                        mode: 'markers',
+                        name: selectedKey,
+                    };
+
+                    const layout = {
+                        xaxis: {
+                            title: 'Iteration Index',
+                            gridcolor: '#e9ecef'
+                        },
+                        yaxis: {
+                            title: selectedKey,
+                            gridcolor: '#e9ecef'
+                        },
+                        plot_bgcolor: '#ffffff',
+                        paper_bgcolor: '#ffffff',
+                        margin: { t: 60, r: 40, b: 60, l: 80 }
+                    };
+
+                    const config = {
+                        responsive: true,
+                        displayModeBar: true,
+                        modeBarButtonsToRemove: ['lasso2d', 'select2d']
+                    };
+
+                    Plotly.newPlot('phase-plot', [trace], layout, config);
+                }
+
+                select.addEventListener('change', e => {
+                    if (e.target.value) {
+                        plotData(e.target.value);
+                    }
+                });
+
+                // Plot first available data type
+                if (allKeys.size > 0) {
+                    plotData([...allKeys][0]);
+                }
+            })
+            .catch(error => {
+                loadingSpinner.style.display = 'none';
+                console.error('Error loading phase data:', error);
+
+                const dataSection = document.querySelector('.data-section');
+                // Hide the section on error as well
+                dataSection.style.display = 'none';
+
+                // Optionally, you could show an error message instead:
+                // dataSection.innerHTML = `
+                //     <div class="alert alert-danger m-3" role="alert">
+                //         <i class="fas fa-exclamation-triangle me-2"></i>
+                //         <strong>Error:</strong> Unable to load phase data. ${error.message}
+                //     </div>
+                // `;
+            });
+    });
+</script>
+
+{% endblock %}