ivoryos 0.1.20__py3-none-any.whl → 0.1.22__py3-none-any.whl

ivoryos/__init__.py

@@ -1,10 +1,8 @@
-import importlib
-import inspect
 import os
 import sys
 from typing import Union
 
-from flask import Flask, redirect, url_for, Blueprint, g
+from flask import Flask, redirect, url_for, g
 
 from ivoryos.config import Config, get_config
 from ivoryos.routes.auth.auth import auth, login_manager
@@ -20,14 +18,30 @@ from ivoryos.utils.script_runner import ScriptRunner
 from ivoryos.version import __version__ as ivoryos_version
 from importlib.metadata import entry_points
 global_config = GlobalConfig()
+from sqlalchemy import event
+from sqlalchemy.engine import Engine
+import sqlite3
+
+@event.listens_for(Engine, "connect")
+def enforce_sqlite_foreign_keys(dbapi_connection, connection_record):
+    if isinstance(dbapi_connection, sqlite3.Connection):
+        cursor = dbapi_connection.cursor()
+        cursor.execute("PRAGMA foreign_keys=ON")
+        cursor.close()
 
 url_prefix = os.getenv('URL_PREFIX', "/ivoryos")
 app = Flask(__name__, static_url_path=f'{url_prefix}/static', static_folder='static')
+app.register_blueprint(main, url_prefix=url_prefix)
+app.register_blueprint(auth, url_prefix=url_prefix)
+app.register_blueprint(control, url_prefix=url_prefix)
+app.register_blueprint(design, url_prefix=url_prefix)
+app.register_blueprint(database, url_prefix=url_prefix)
 
 
 def create_app(config_class=None):
-    # url_prefix = os.getenv('URL_PREFIX', "/ivoryos")
-    # app = Flask(__name__, static_url_path=f'{url_prefix}/static', static_folder='static')
+    """
+    create app, init database
+    """
     app.config.from_object(config_class or 'config.get_config()')
 
     # Initialize extensions
@@ -82,18 +96,9 @@ def run(module=None, host="0.0.0.0", port=None, debug=None, llm_server=None, mod
     :param logger: logger name of list of logger names, defaults to None
     :param logger_output_name: log file save name of logger, defaults to None, and will use "default.log"
     :param enable_design: enable design canvas, database and workflow execution
-    :param stream_address:
     """
     app = create_app(config_class=config or get_config())  # Create app instance using factory function
 
-    app.register_blueprint(main, url_prefix=url_prefix)
-    app.register_blueprint(auth, url_prefix=url_prefix)
-    app.register_blueprint(control, url_prefix=url_prefix)
-
-    if enable_design:
-        app.register_blueprint(design, url_prefix=url_prefix)
-        app.register_blueprint(database, url_prefix=url_prefix)
-
     plugins = load_plugins(app, socketio)
 
     def inject_nav_config():
@@ -115,10 +120,8 @@ def run(module=None, host="0.0.0.0", port=None, debug=None, llm_server=None, mod
         app.config["MODULE"] = module
         app.config["OFF_LINE"] = False
         global_config.deck = sys.modules[module]
-        # global_config.heinsight = HeinsightAPI("http://127.0.0.1:8080")
         global_config.deck_snapshot = utils.create_deck_snapshot(global_config.deck,
                                                                  output_path=app.config["DUMMY_DECK"], save=True)
-        # global_config.runner = ScriptRunner(globals())
     else:
         app.config["OFF_LINE"] = True
     if model:
@@ -156,3 +159,5 @@ def load_plugins(app, socketio):
         app.register_blueprint(getattr(plugin, entry_point.name), url_prefix=f"{url_prefix}/{entry_point.name}")
 
     return plugin_names
+
+

ivoryos/routes/control/control.py

@@ -151,7 +151,7 @@ def controllers(instrument: str):
     return render_template('controllers.html', instrument=instrument, forms=forms, format_name=format_name)
 
 
-@control.route("/backend_control/<instrument>", methods=['GET', 'POST'])
+@control.route("/backend_control/<instrument>", methods=['POST'])
 def backend_control(instrument: str=None):
     """
     .. :quickref: Backend Control; backend control
@@ -187,7 +187,7 @@ def backend_control(instrument: str=None):
                 return json_output, 400
         else:
             return "instrument not exist", 400
-    return json_output, 200
+        return json_output, 200
 
 
 @control.route("/backend_control", methods=['GET'])

ivoryos/routes/control/templates/control/controllers_home.html

@@ -9,7 +9,7 @@
 {#                    {% if not deck %}#}
 {#                        <a href="{{ url_for('control.disconnect', instrument=instrument) }}" class="stretched-link controller-card" style="float: right;color: red; position: relative;">Disconnect <i class="bi bi-x-square"></i></a>#}
                         <div class="p-4 controller-card">
-                            <h5 class=""><a href="{{ url_for('control.controllers', instrument=instrument) }}" class="text-dark stretched-link">{{instrument}}</a></h5>
+                            <h5 class=""><a href="{{ url_for('control.controllers', instrument=instrument) }}" class="text-dark stretched-link">{{instrument.split(".")[1]}}</a></h5>
                         </div>
 {#                    {% else %}#}
 {#                        <div class="p-4 controller-card">#}
@@ -19,6 +19,11 @@
                 </div>
             </div>
         {% endfor %}
+        <div class="d-flex mb-3">
+            <a href="{{ url_for('design.download', filetype='proxy') }}" class="btn btn-outline-primary">
+                <i class="bi bi-download"></i> Download remote control script
+            </a>
+        </div>
         {% if not deck %}
             <div class="col-xl-3 col-lg-4 col-md-6 mb-4 ">
                 <div class="bg-white rounded shadow-sm position-relative">

ivoryos/routes/database/database.py

@@ -1,7 +1,7 @@
-from flask import Blueprint, redirect, url_for, flash, request, render_template, session, current_app
+from flask import Blueprint, redirect, url_for, flash, request, render_template, session, current_app, jsonify
 from flask_login import login_required
 
-from ivoryos.utils.db_models import Script, db
+from ivoryos.utils.db_models import Script, db, WorkflowRun, WorkflowStep
 from ivoryos.utils.utils import get_script_file, post_script_file
 
 database = Blueprint('database', __name__, template_folder='templates/database')
@@ -176,13 +176,56 @@ def save_as():
     """
     if request.method == "POST":
         run_name = request.form.get("run_name")
+        register_workflow = request.form.get("register_workflow")
         exist_script = Script.query.get(run_name)
         if not exist_script:
             script = get_script_file()
             script.save_as(run_name)
+            script.registered = register_workflow == "on"
             script.author = session.get('user')
             post_script_file(script)
             publish()
         else:
             flash("Script name is already exist in database")
         return redirect(url_for("design.experiment_builder"))
+
+
+@database.route('/workflow_runs')
+def list_workflows():
+    query = WorkflowRun.query
+    search_term = request.args.get("keyword", None)
+    if search_term:
+        query = query.filter(WorkflowRun.name.like(f'%{search_term}%'))
+    page = request.args.get('page', default=1, type=int)
+    per_page = 10
+
+    workflows = query.paginate(page=page, per_page=per_page, error_out=False)
+    return render_template('workflow_run_database.html', workflows=workflows)
+
+
+@database.route('/workflow_steps/<int:workflow_id>')
+def get_workflow_steps(workflow_id):
+    steps = WorkflowStep.query.filter_by(workflow_id=workflow_id).all()
+    steps_data = [step.as_dict() for step in steps]
+    return jsonify({'steps': steps_data})
+
+
+@database.route("/delete_workflow_data/<workflow_id>")
+@login_required
+def delete_workflow_data(workflow_id: str):
+    """
+    .. :quickref: Database; delete experiment data from database
+
+    delete workflow data from database
+
+    .. http:get:: /delete_workflow_data/<workflow_id>
+
+    :param workflow_id: workflow id
+    :type workflow_id: str
+    :status 302: redirect to :http:get:`/ivoryos/workflow_runs/`
+
+    """
+    run = WorkflowRun.query.get(workflow_id)
+    db.session.delete(run)
+    db.session.commit()
+    return redirect(url_for('database.list_workflows'))

ivoryos/routes/database/templates/database/experiment_database.html

@@ -26,10 +26,11 @@
             <tr>
                 <th scope="col">Workflow name</th>
                 <th scope="col">Deck </th>
-                <th scope="col">Current status</th>
+                <th scope="col">Editing</th>
                 <th scope="col">Time created</th>
                 <th scope="col">Last modified</th>
                 <th scope="col">Author</th>
+{#                <th scope="col">Registered</th>#}
                 <th scope="col"></th>
             </tr>
         </thead>
@@ -42,12 +43,13 @@
                     <td>{{ workflow.time_created }}</td>
                     <td>{{ workflow.last_modified }}</td>
                     <td>{{ workflow.author }}</td>
+{#                    <td>{{ workflow.registered }}</td>#}
                     <td>
                         {#not workflow.status == "finalized" or#}
                         {% if session['user'] == 'admin' or session['user'] == workflow.author %}
                             <a href="{{ url_for('database.delete_workflow', workflow_name=workflow.name) }}">delete</a>
                         {% else %}
-                            <a class="disabled-link" href="{{ url_for('database.delete_workflow', workflow_name=workflow.name) }}">delete</a>
+                            <a class="disabled-link">delete</a>
                         {% endif %}
                     <td>
                 </tr>

ivoryos/routes/database/templates/database/workflow_run_database.html

@@ -0,0 +1,81 @@
+{% extends 'base.html' %}
+
+{% block title %}IvoryOS | Design Database{% endblock %}
+{% block body %}
+
+        <table class="table table-hover" id="workflowResultLibrary">
+        <thead>
+            <tr>
+                <th scope="col">Workflow name</th>
+                <th scope="col">Start time</th>
+                <th scope="col">End time</th>
+                <th scope="col">Data</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for workflow in workflows %}
+                <tr>
+                    <td><a href="{{ url_for('database.get_workflow_steps', workflow_id=workflow.id) }}">{{ workflow.name }}</a></td>
+                    <td>{{ workflow.start_time.strftime("%Y-%m-%d %H:%M:%S") if workflow.start_time else '' }}</td>
+                    <td>{{ workflow.end_time.strftime("%Y-%m-%d %H:%M:%S") if workflow.end_time else '' }}</td>
+
+                    <td>
+                        {% if workflow.data_path %}
+                          <a href="{{ url_for('design.download_results', filename=workflow.data_path) }}">{{ workflow.data_path }}</a>
+                        {% endif %}
+                    </td>
+                    <td>
+                        {% if session['user'] == 'admin' or session['user'] == workflow.author %}
+                            <a href="{{ url_for('database.delete_workflow_data', workflow_id=workflow.id) }}">delete</a>
+                        {% else %}
+                            <a class="disabled-link">delete</a>
+                        {% endif %}
+                    </td>
+                </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+
+{#    paging#}
+    <div class="pagination justify-content-center">
+        <div class="page-item {{ 'disabled' if not workflows.has_prev else '' }}">
+            <a class="page-link" href="{{ url_for('database.list_workflows', page=workflows.prev_num) }}">Previous</a>
+        </div>
+        {% for num in workflows.iter_pages() %}
+            <div class="page-item">
+                <a class="page-link {{ 'active' if num == workflows.page else '' }}" href="{{ url_for('database.list_workflows', page=num) }}">{{ num }}</a>
+            </div>
+        {% endfor %}
+        <div class="page-item {{ 'disabled' if not workflows.has_next else '' }}">
+            <a class="page-link" href="{{ url_for('database.list_workflows', page=workflows.next_num) }}">Next</a>
+        </div>
+    </div>
+
+    <div id="steps-container"></div>
+
+    <script>
+        function showSteps(workflowId) {
+            fetch(`/workflow_steps/${workflowId}`)
+                .then(response => response.json())
+                .then(data => {
+                    const container = document.getElementById('steps-container');
+                    container.innerHTML = '';  // Clear previous content
+                    const stepsList = document.createElement('ul');
+
+                    data.steps.forEach(step => {
+                        const li = document.createElement('li');
+                        li.innerHTML = `
+                            <strong>Step: </strong> ${step.method_name} <br>
+                            <strong>Start Time:</strong> ${step.start_time} <br>
+                            <strong>End Time:</strong> ${step.end_time} <br>
+                            <strong>Human Intervention:</strong> ${step.run_error ? 'Yes' : 'No'}
+                        `;
+                        stepsList.appendChild(li);
+                    });
+
+                    container.appendChild(stepsList);
+                });
+        }
+    </script>
+
+{% endblock %}

ivoryos/routes/design/design.py

@@ -11,11 +11,13 @@ from flask_socketio import SocketIO
 from werkzeug.utils import secure_filename
 
 from ivoryos.utils import utils
+from ivoryos.utils.client_proxy import create_function, export_to_python
 from ivoryos.utils.global_config import GlobalConfig
 from ivoryos.utils.form import create_builtin_form, create_action_button, format_name, create_form_from_pseudo, \
-    create_form_from_action
+    create_form_from_action, create_all_builtin_forms
 from ivoryos.utils.db_models import Script
 from ivoryos.utils.script_runner import ScriptRunner
+# from ivoryos.utils.utils import load_workflows
 
 socketio = SocketIO()
 design = Blueprint('design', __name__, template_folder='templates/design')
@@ -92,6 +94,9 @@ def experiment_builder(instrument=None):
     """
     deck = global_config.deck
     script = utils.get_script_file()
+    # load_workflows(script)
+    # registered_workflows = global_config.registered_workflows
+
     if deck and script.deck is None:
         script.deck = os.path.splitext(os.path.basename(deck.__file__))[
             0] if deck.__name__ == "__main__" else deck.__name__
@@ -114,7 +119,10 @@ def experiment_builder(instrument=None):
 
     functions = {}
     if deck:
-        deck_variables = global_config.deck_snapshot.keys()
+        deck_variables = list(global_config.deck_snapshot.keys())
+        # deck_variables.insert(0, "registered_workflows")
+        deck_variables.insert(0, "flow_control")
+
     else:
         deck_variables = list(pseudo_deck.keys()) if pseudo_deck else []
         deck_variables.remove("deck_name") if len(deck_variables) > 0 else deck_variables
@@ -122,23 +130,31 @@ def experiment_builder(instrument=None):
     if edit_action_info:
         forms = create_form_from_action(edit_action_info, script=script)
     elif instrument:
-        if instrument in ['if', 'while', 'variable', 'wait', 'repeat']:
-            forms = create_builtin_form(instrument, script=script)
+        # if instrument in ['if', 'while', 'variable', 'wait', 'repeat']:
+        #     forms = create_builtin_form(instrument, script=script)
+        if instrument == 'flow_control':
+            forms = create_all_builtin_forms(script=script)
+        # elif instrument == 'registered_workflows':
+        #     functions = utils._inspect_class(registered_workflows)
+        #     # forms = create_workflow_forms(script=script)
+        #     forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
+        elif instrument in global_config.defined_variables.keys():
+            _object = global_config.defined_variables.get(instrument)
+            functions = utils._inspect_class(_object)
+            forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
         else:
             if deck:
                 functions = global_config.deck_snapshot.get(instrument, {})
             elif pseudo_deck:
                 functions = pseudo_deck.get(instrument, {})
-            # print(function_metadata)
-            # functions = {key: data.get('signature', {}) for key, data in function_metadata.items()}
             forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
         if request.method == 'POST' and "hidden_name" in request.form:
             # all_kwargs = request.form.copy()
             method_name = request.form.get("hidden_name", None)
             # if method_name is not None:
             form = forms.get(method_name)
+            insert_position = request.form.get("drop_target_id", None)
             kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
-
             if form and form.validate_on_submit():
                 function_name = kwargs.pop("hidden_name")
                 save_data = kwargs.pop('return', '')
@@ -151,30 +167,57 @@ def experiment_builder(instrument=None):
                           "args": kwargs,
                           "return": save_data,
                           'arg_types': primitive_arg_types}
-                script.add_action(action=action)
+                script.add_action(action=action, insert_position=insert_position)
             else:
                 flash(form.errors)
 
         elif request.method == 'POST' and "builtin_name" in request.form:
-            kwargs = {field.name: field.data for field in forms if field.name != 'csrf_token'}
-            if forms.validate_on_submit():
+            function_name = request.form.get("builtin_name")
+            form = forms.get(function_name)
+            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
+            insert_position = request.form.get("drop_target_id", None)
+
+            if form.validate_on_submit():
                 # print(kwargs)
                 logic_type = kwargs.pop('builtin_name')
                 if 'variable' in kwargs:
                     try:
-                        script.add_variable(**kwargs)
+                        script.add_variable(**kwargs, insert_position=insert_position)
                     except ValueError:
                         flash("Invalid variable type")
                 else:
-                    script.add_logic_action(logic_type=logic_type, **kwargs)
+                    script.add_logic_action(logic_type=logic_type, **kwargs, insert_position=insert_position)
             else:
-                flash(forms.errors)
+                flash(form.errors)
+        elif request.method == 'POST' and "workflow_name" in request.form:
+            workflow_name = request.form.get("workflow_name")
+            form = forms.get(workflow_name)
+            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
+            insert_position = request.form.get("drop_target_id", None)
+
+            if form.validate_on_submit():
+                # workflow_name = kwargs.pop('workflow_name')
+                save_data = kwargs.pop('return', '')
+
+                primitive_arg_types = utils.get_arg_type(kwargs, functions[workflow_name])
 
-        # toggle autofill
+                script.eval_list(kwargs, primitive_arg_types)
+                kwargs = script.validate_variables(kwargs)
+                action = {"instrument": instrument, "action": workflow_name,
+                          "args": kwargs,
+                          "return": save_data,
+                          'arg_types': primitive_arg_types}
+                script.add_action(action=action, insert_position=insert_position)
+                script.add_workflow(**kwargs, insert_position=insert_position)
+            else:
+                flash(form.errors)
+
+        # toggle autofill, autofill doesn't apply to control flow ops
         elif request.method == 'POST' and "autofill" in request.form:
             autofill = not autofill
-            forms = create_form_from_pseudo(functions, autofill=autofill, script=script)
             session['autofill'] = autofill
+            if not instrument == 'flow_control':
+                forms = create_form_from_pseudo(functions, autofill=autofill, script=script)
 
     utils.post_script_file(script)
     design_buttons = create_action_button(script)
@@ -318,7 +361,7 @@ def experiment_run():
             run_name = script.validate_function_name(run_name)
             runner.run_script(script=script, run_name=run_name, config=config, bo_args=bo_args,
                               logger=g.logger, socketio=g.socketio, repeat_count=repeat,
-                              output_path=datapath
+                              output_path=datapath, current_app=current_app._get_current_object()
                               )
             if utils.check_config_duplicate(config):
                 flash(f"WARNING: Duplicate in config entries.")
@@ -329,7 +372,7 @@ def experiment_run():
                            return_list=return_list, config_list=config_list, config_file_list=config_file_list,
                            config_preview=config_preview, data_list=data_list, config_type_list=config_type_list,
                            no_deck_warning=no_deck_warning, dismiss=dismiss, design_buttons=design_buttons,
-                           history=deck_list)
+                           history=deck_list, pause_status=runner.pause_status())
 
 
 @design.route("/toggle_script_type/<stype>")
@@ -483,7 +526,20 @@ def download(filetype):
             outfile.write(json_object)
     elif filetype == "python":
         filepath = os.path.join(current_app.config["SCRIPT_FOLDER"], f"{run_name}.py")
-
+    elif filetype == "proxy":
+        snapshot = global_config.deck_snapshot.copy()
+        class_definitions = {}
+        # Iterate through each instrument in the snapshot
+        for instrument_key, instrument_data in snapshot.items():
+            # Iterate through each function associated with the current instrument
+            for function_key, function_data in instrument_data.items():
+                # Convert the function signature to a string representation
+                function_data['signature'] = str(function_data['signature'])
+            class_name = instrument_key.split('.')[-1]  # Extracting the class name from the path
+            class_definitions[class_name.capitalize()] = create_function(request.url_root, class_name, instrument_data)
+        # Export the generated class definitions to a .py script
+        export_to_python(class_definitions, current_app.config["OUTPUT_FOLDER"])
+        filepath = os.path.join(current_app.config["OUTPUT_FOLDER"], "generated_proxy.py")
     return send_file(os.path.abspath(filepath), as_attachment=True)