ivoryos 1.0.9__py3-none-any.whl → 1.4.4__py3-none-any.whl

ivoryos/routes/design/design.py

@@ -1,786 +1,477 @@
-import csv
-import json
 import os
-import pickle
-import sys
-import time
 
-from flask import Blueprint, redirect, url_for, flash, jsonify, send_file, request, render_template, session, \
-    current_app, g
-from flask_login import login_required
-from flask_socketio import SocketIO
-from werkzeug.utils import secure_filename
+from flask import Blueprint, redirect, url_for, flash, jsonify, request, render_template, session, current_app
+from flask_login import login_required, current_user
 
+from ivoryos.routes.library.library import publish
 from ivoryos.utils import utils
 from ivoryos.utils.global_config import GlobalConfig
-from ivoryos.utils.form import create_builtin_form, create_action_button, format_name, create_form_from_pseudo, \
-    create_form_from_action, create_all_builtin_forms
-from ivoryos.utils.db_models import Script, WorkflowRun, SingleStep, WorkflowStep
-from ivoryos.utils.script_runner import ScriptRunner
-# from ivoryos.utils.utils import load_workflows
+from ivoryos.utils.form import create_action_button, create_form_from_pseudo, create_all_builtin_forms
+from ivoryos.utils.db_models import Script
+from ivoryos.utils.py_to_json import convert_to_cards
 
-socketio = SocketIO()
-design = Blueprint('design', __name__, template_folder='templates/design')
+# Import the new modular components
+from ivoryos.routes.design.design_file import files
+from ivoryos.routes.design.design_step import steps
 
-global_config = GlobalConfig()
-runner = ScriptRunner()
-
-def abort_pending():
-    runner.abort_pending()
-    socketio.emit('log', {'message': "aborted pending iterations"})
-
-def abort_current():
-    runner.stop_execution()
-    socketio.emit('log', {'message': "stopped next task"})
-
-def pause():
-    runner.retry = False
-    msg = runner.toggle_pause()
-    socketio.emit('log', {'message': msg})
-    return msg
-
-def retry():
-    runner.retry = True
-    msg = runner.toggle_pause()
-    socketio.emit('log', {'message': msg})
-
-
-# ---- Socket.IO Event Handlers ----
-
-@socketio.on('abort_pending')
-def handle_abort_pending():
-    abort_pending()
 
-@socketio.on('abort_current')
-def handle_abort_current():
-    abort_current()
+design = Blueprint('design', __name__, template_folder='templates')
 
-@socketio.on('pause')
-def handle_pause():
-    pause()
-
-@socketio.on('retry')
-def handle_retry():
-    retry()
+# Register sub-blueprints
+design.register_blueprint(files)
+design.register_blueprint(steps)
 
+global_config = GlobalConfig()
 
-@socketio.on('connect')
-def handle_abort_action():
-    # Fetch log messages from local file
-    filename = os.path.join(current_app.config["OUTPUT_FOLDER"], current_app.config["LOGGERS_PATH"])
-    with open(filename, 'r') as log_file:
-        log_history = log_file.readlines()
-    for message in log_history[-10:]:
-        socketio.emit('log', {'message': message})
+# ---- Main Design Routes ----
 
 
-@design.route("/design/script/", methods=['GET', 'POST'])
-@design.route("/design/script/<instrument>/", methods=['GET', 'POST'])
+def _create_forms(instrument, script, autofill, pseudo_deck = None):
+    deck = global_config.deck
+    functions = {}
+    if instrument == 'flow_control':
+        forms = create_all_builtin_forms(script=script)
+    elif instrument in global_config.defined_variables.keys():
+        _object = global_config.defined_variables.get(instrument)
+        functions = utils._inspect_class(_object)
+        forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
+    elif instrument.startswith("blocks"):
+        forms = create_form_from_pseudo(pseudo=global_config.building_blocks[instrument], autofill=autofill, script=script)
+        functions = global_config.building_blocks[instrument]
+    else:
+        if deck:
+            functions = global_config.deck_snapshot.get(instrument, {})
+        elif pseudo_deck:
+            functions = pseudo_deck.get(instrument, {})
+        forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
+    return functions, forms
+
+@design.route("/draft")
 @login_required
-def experiment_builder(instrument=None):
+def experiment_builder():
     """
     .. :quickref: Workflow Design; Build experiment workflow
 
     **Experiment Builder**
 
-    This route allows users to build and edit experiment workflows. Users can interact with available instruments,
-    define variables, and manage experiment scripts.
-
-    .. http:get:: /design/script
+    .. http:get:: /draft
 
-    Load the experiment builder interface.
+    Load the experiment builder page where users can design their workflow by adding actions, instruments, and logic.
 
-    :param instrument: The specific instrument for which to load functions and forms.
-    :type instrument: str
     :status 200: Experiment builder loaded successfully.
 
-    .. http:post:: /design/script
-
-    Submit form data to add or modify actions in the experiment script.
-
-    **Adding action to canvas**
-
-    :form return: (optional) The name of the function or method to add to the script.
-    :form dynamic: depend on the selected instrument and its metadata.
-
-    :status 200: Action added or modified successfully.
-    :status 400: Validation errors in submitted form data.
-    :status 302: Toggles autofill or redirects to refresh the page.
-
-    **Toggle auto parameter name fill**:
-
-    :status 200: autofill toggled successfully
-
     """
     deck = global_config.deck
     script = utils.get_script_file()
-    # load_workflows(script)
-    # registered_workflows = global_config.registered_workflows
 
     if deck and script.deck is None:
         script.deck = os.path.splitext(os.path.basename(deck.__file__))[
             0] if deck.__name__ == "__main__" else deck.__name__
-    # script.sort_actions()
-
+        utils.post_script_file(script)
     pseudo_deck_name = session.get('pseudo_deck', '')
     pseudo_deck_path = os.path.join(current_app.config["DUMMY_DECK"], pseudo_deck_name)
     off_line = current_app.config["OFF_LINE"]
-    enable_llm = current_app.config["ENABLE_LLM"]
-    autofill = session.get('autofill')
 
-    # autofill is not allowed for prep and cleanup
-    autofill = autofill if script.editing_type == "script" else False
-    forms = None
     pseudo_deck = utils.load_deck(pseudo_deck_path) if off_line and pseudo_deck_name else None
     if off_line and pseudo_deck is None:
         flash("Choose available deck below.")
 
     deck_list = utils.available_pseudo_deck(current_app.config["DUMMY_DECK"])
 
-    functions = {}
     if deck:
         deck_variables = list(global_config.deck_snapshot.keys())
-        # deck_variables.insert(0, "registered_workflows")
-        deck_variables.insert(0, "flow_control")
-
+        # deck_variables.insert(0, "flow_control")
     else:
         deck_variables = list(pseudo_deck.keys()) if pseudo_deck else []
         deck_variables.remove("deck_name") if len(deck_variables) > 0 else deck_variables
-    edit_action_info = session.get("edit_action")
-    if edit_action_info:
-        forms = create_form_from_action(edit_action_info, script=script)
-    elif instrument:
-        # if instrument in ['if', 'while', 'variable', 'wait', 'repeat']:
-        #     forms = create_builtin_form(instrument, script=script)
-        if instrument == 'flow_control':
-            forms = create_all_builtin_forms(script=script)
-        # elif instrument == 'registered_workflows':
-        #     functions = utils._inspect_class(registered_workflows)
-        #     # forms = create_workflow_forms(script=script)
-        #     forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
-        elif instrument in global_config.defined_variables.keys():
-            _object = global_config.defined_variables.get(instrument)
-            functions = utils._inspect_class(_object)
-            forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
-        else:
-            if deck:
-                functions = global_config.deck_snapshot.get(instrument, {})
-            elif pseudo_deck:
-                functions = pseudo_deck.get(instrument, {})
-            forms = create_form_from_pseudo(pseudo=functions, autofill=autofill, script=script)
-        if request.method == 'POST' and "hidden_name" in request.form:
-            # all_kwargs = request.form.copy()
-            method_name = request.form.get("hidden_name", None)
-            # if method_name is not None:
-            form = forms.get(method_name)
-            insert_position = request.form.get("drop_target_id", None)
-            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
-            if form and form.validate_on_submit():
-                function_name = kwargs.pop("hidden_name")
-                save_data = kwargs.pop('return', '')
-
-                primitive_arg_types = utils.get_arg_type(kwargs, functions[function_name])
-
-                script.eval_list(kwargs, primitive_arg_types)
-                kwargs = script.validate_variables(kwargs)
-                action = {"instrument": instrument, "action": function_name,
-                          "args": kwargs,
-                          "return": save_data,
-                          'arg_types': primitive_arg_types}
-                script.add_action(action=action, insert_position=insert_position)
-            else:
-                flash(form.errors)
-
-        elif request.method == 'POST' and "builtin_name" in request.form:
-            function_name = request.form.get("builtin_name")
-            form = forms.get(function_name)
-            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
-            insert_position = request.form.get("drop_target_id", None)
-
-            if form.validate_on_submit():
-                # print(kwargs)
-                logic_type = kwargs.pop('builtin_name')
-                if 'variable' in kwargs:
-                    try:
-                        script.add_variable(insert_position=insert_position, **kwargs)
-                    except ValueError:
-                        flash("Invalid variable type")
-                else:
-                    script.add_logic_action(logic_type=logic_type, insert_position=insert_position, **kwargs)
-            else:
-                flash(form.errors)
-        elif request.method == 'POST' and "workflow_name" in request.form:
-            workflow_name = request.form.get("workflow_name")
-            form = forms.get(workflow_name)
-            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
-            insert_position = request.form.get("drop_target_id", None)
-
-            if form.validate_on_submit():
-                # workflow_name = kwargs.pop('workflow_name')
-                save_data = kwargs.pop('return', '')
-
-                primitive_arg_types = utils.get_arg_type(kwargs, functions[workflow_name])
-
-                script.eval_list(kwargs, primitive_arg_types)
-                kwargs = script.validate_variables(kwargs)
-                action = {"instrument": instrument, "action": workflow_name,
-                          "args": kwargs,
-                          "return": save_data,
-                          'arg_types': primitive_arg_types}
-                script.add_action(action=action, insert_position=insert_position)
-                script.add_workflow(**kwargs, insert_position=insert_position)
-            else:
-                flash(form.errors)
-
-        # toggle autofill, autofill doesn't apply to control flow ops
-        elif request.method == 'POST' and "autofill" in request.form:
-            autofill = not autofill
-            session['autofill'] = autofill
-            if not instrument == 'flow_control':
-                forms = create_form_from_pseudo(functions, autofill=autofill, script=script)
 
-    utils.post_script_file(script)
+    # edit_action_info = session.get("edit_action")
 
-    exec_string = script.python_script if script.python_script else script.compile(current_app.config['SCRIPT_FOLDER'])
+    try:
+        exec_string = script.python_script if script.python_script else script.compile(current_app.config['SCRIPT_FOLDER'])
+    except Exception as e:
+        exec_string = {}
+        flash(f"Error in Python script: {e}")
     session['python_code'] = exec_string
 
-    design_buttons = create_action_button(script)
-    return render_template('experiment_builder.html', off_line=off_line, instrument=instrument, history=deck_list,
-                           script=script, defined_variables=deck_variables,
-                           local_variables=global_config.defined_variables,
-                           forms=forms, buttons=design_buttons, format_name=format_name,
-                           use_llm=enable_llm)
+    design_buttons = {stype: create_action_button(script, stype) for stype in script.stypes}
 
+    return render_template('experiment_builder.html', off_line=off_line, history=deck_list,
+                           script=script, defined_variables=deck_variables, buttons_dict=design_buttons,
+                           local_variables=global_config.defined_variables, block_variables=global_config.building_blocks)
 
-@design.route("/design/generate_code", methods=['POST'])
+@design.route("/draft/code_preview", methods=["GET"])
 @login_required
-def generate_code():
-    """
-    .. :quickref: Text to Code; Generate code from user input and update the design canvas.
+def compile_preview():
+    # Get mode and batch from query parameters
+    script = utils.get_script_file()
+    mode = request.args.get("mode", "single")   # default to "single"
+    batch = request.args.get("batch", "sample") # default to "sample"
 
-    .. http:post:: /design/generate_code
+    try:
+        # Example: decide which code to return based on mode/batch
+        if mode == "single":
+            code = script.compile(current_app.config['SCRIPT_FOLDER'])
+        elif mode == "batch":
+            code = script.compile(current_app.config['SCRIPT_FOLDER'], batch=True, mode=batch)
+        else:
+            code = "Invalid mode. Please select 'single' or 'batch'."
+    except Exception as e:
+        code = f"Error compiling: {e}"
+    # print(code)
+    return jsonify(code=code)
 
-    :form prompt: user's prompt
-    :status 200: and then redirects to :http:get:`/experiment/build`
-    :status 400: failed to initialize the AI agent redirects to :http:get:`/design/script`
 
-    """
-    agent = global_config.agent
-    enable_llm = current_app.config["ENABLE_LLM"]
-    instrument = request.form.get("instrument")
-
-    if request.method == 'POST' and "clear" in request.form:
-        session['prompt'][instrument] = ''
-    if request.method == 'POST' and "gen" in request.form:
-        prompt = request.form.get("prompt")
-        session['prompt'][instrument] = prompt
-        # sdl_module = utils.parse_functions(find_instrument_by_name(f'deck.{instrument}'), doc_string=True)
-        sdl_module = global_config.deck_snapshot.get(instrument, {})
-        empty_script = Script(author=session.get('user'))
-        if enable_llm and agent is None:
-            try:
-                model = current_app.config["LLM_MODEL"]
-                server = current_app.config["LLM_SERVER"]
-                module = current_app.config["MODULE"]
-                from ivoryos.utils.llm_agent import LlmAgent
-                agent = LlmAgent(host=server, model=model, output_path=os.path.dirname(os.path.abspath(module)))
-            except Exception as e:
-                flash(e.__str__())
-                return redirect(url_for("design.experiment_builder", instrument=instrument, use_llm=True)), 400
-        action_list = agent.generate_code(sdl_module, prompt)
-        for action in action_list:
-            action['instrument'] = instrument
-            action['return'] = ''
-            if "args" not in action:
-                action['args'] = {}
-            if "arg_types" not in action:
-                action['arg_types'] = {}
-            empty_script.add_action(action)
-        utils.post_script_file(empty_script)
-    return redirect(url_for("design.experiment_builder", instrument=instrument, use_llm=True))
-
-
-@design.route("/design/campaign", methods=['GET', 'POST'])
+@design.route("/draft/meta", methods=["PATCH"])
 @login_required
-def experiment_run():
+def update_script_meta():
     """
-    .. :quickref: Workflow Execution; Execute/iterate the workflow
-
-    .. http:get:: /design/campaign
+    .. :quickref: Workflow Design; update the script metadata.
 
-    Compile the workflow and load the experiment execution interface.
+    .. http:patch:: /draft/meta
 
-    .. http:post:: /design/campaign
+    Update the script metadata, including the script name and status. If the script name is provided,
+    it saves the script with that name. If the status is "finished", it finalizes the script.
 
-    Start workflow execution
+    :form name: The name to save the script as.
+    :form status: The status of the script (e.g., "finished").
 
+    :status 200: Successfully updated the script metadata.
     """
-    deck = global_config.deck
-    script = utils.get_script_file()
-
-    # script.sort_actions() # handled in update list
-    off_line = current_app.config["OFF_LINE"]
-    deck_list = utils.import_history(os.path.join(current_app.config["OUTPUT_FOLDER"], 'deck_history.txt'))
-    # if not off_line and deck is None:
-    #     # print("loading deck")
-    #     module = current_app.config.get('MODULE', '')
-    #     deck = sys.modules[module] if module else None
-    #     script.deck = os.path.splitext(os.path.basename(deck.__file__))[0]
-    design_buttons = {stype: create_action_button(script, stype) for stype in script.stypes}
-    config_preview = []
-    config_file_list = [i for i in os.listdir(current_app.config["CSV_FOLDER"]) if not i == ".gitkeep"]
-    try:
-        # todo
-        exec_string = script.python_script if script.python_script else script.compile(current_app.config['SCRIPT_FOLDER'])
-        # exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
-        # print(exec_string)
-    except Exception as e:
-        flash(e.__str__())
-        # handle api request
-        if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
-            return jsonify({"error": e.__str__()})
-        else:
-            return redirect(url_for("design.experiment_builder"))
-
-    config_file = request.args.get("filename")
-    config = []
-    if config_file:
-        session['config_file'] = config_file
-    filename = session.get("config_file")
-    if filename:
-        # config_preview = list(csv.DictReader(open(os.path.join(current_app.config['CSV_FOLDER'], filename))))
-        config = list(csv.DictReader(open(os.path.join(current_app.config['CSV_FOLDER'], filename))))
-        config_preview = config[1:]
-        arg_type = config.pop(0)  # first entry is types
-    try:
-        for key, func_str in exec_string.items():
-            exec(func_str)
-        line_collection = script.convert_to_lines(exec_string)
-
-    except Exception:
-        flash(f"Please check {key} syntax!!")
-        return redirect(url_for("design.experiment_builder"))
-    # runner.globals_dict.update(globals())
-    run_name = script.name if script.name else "untitled"
-
-    dismiss = session.get("dismiss", None)
+    data = request.get_json()
     script = utils.get_script_file()
-    no_deck_warning = False
-
-    _, return_list = script.config_return()
-    config_list, config_type_list = script.config("script")
-    # config = script.config("script")
-    data_list = os.listdir(current_app.config['DATA_FOLDER'])
-    data_list.remove(".gitkeep") if ".gitkeep" in data_list else data_list
-    if deck is None:
-        no_deck_warning = True
-        flash(f"No deck is found, import {script.deck}")
-    elif script.deck:
-        is_deck_match = script.deck == deck.__name__ or script.deck == \
-                        os.path.splitext(os.path.basename(deck.__file__))[0]
-        if not is_deck_match:
-            flash(f"This script is not compatible with current deck, import {script.deck}")
-    if request.method == "POST":
-        bo_args = None
-        compiled = False
-        if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
-            payload_json = request.get_json()
-            compiled = True
-            if "kwargs" in payload_json:
-                config = payload_json["kwargs"]
-            elif "parameters" in payload_json:
-                bo_args = payload_json
-            repeat = payload_json.pop("repeat", None)
+    if 'name' in data:
+        run_name = data.get("name")
+        exist_script = Script.query.get(run_name)
+        if exist_script is None:
+            script.save_as(run_name)
+            utils.post_script_file(script)
+            return jsonify(success=True)
         else:
-            if "bo" in request.form:
-                bo_args = request.form.to_dict()
-            if "online-config" in request.form:
-                config = utils.web_config_entry_wrapper(request.form.to_dict(), config_list)
-            repeat = request.form.get('repeat', None)
+            flash("Script name is already exist in database")
+            return jsonify(success=False)
 
-        try:
-            datapath = current_app.config["DATA_FOLDER"]
-            run_name = script.validate_function_name(run_name)
-            runner.run_script(script=script, run_name=run_name, config=config, bo_args=bo_args,
-                              logger=g.logger, socketio=g.socketio, repeat_count=repeat,
-                              output_path=datapath, compiled=compiled,
-                              current_app=current_app._get_current_object()
-                              )
-            if utils.check_config_duplicate(config):
-                flash(f"WARNING: Duplicate in config entries.")
-        except Exception as e:
-            if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
-                return jsonify({"error": e.__str__()})
-            else:
-                flash(e)
-    if request.accept_mimetypes.best_match(['application/json', 'text/html']) == 'application/json':
-        # wait to get a workflow ID
-        while not global_config.runner_status:
-            time.sleep(1)
-        return jsonify({"status": "task started", "task_id": global_config.runner_status.get("id")})
-    else:
-        return render_template('experiment_run.html', script=script.script_dict, filename=filename,
-                           dot_py=exec_string, line_collection=line_collection,
-                           return_list=return_list, config_list=config_list, config_file_list=config_file_list,
-                           config_preview=config_preview, data_list=data_list, config_type_list=config_type_list,
-                           no_deck_warning=no_deck_warning, dismiss=dismiss, design_buttons=design_buttons,
-                           history=deck_list, pause_status=runner.pause_status())
+    if 'status' in data:
+        if data['status'] == "finished":
+            script.finalize()
+            utils.post_script_file(script)
+            return jsonify(success=True)
+    return jsonify(success=False)
 
 
-@design.route("/design/script/toggle/<stype>")
+@design.route("/draft/ui-state", methods=["PATCH"])
 @login_required
-def toggle_script_type(stype=None):
+def update_ui_state():
     """
-    .. :quickref: Workflow Design; toggle the experimental phase for design canvas.
-
-    .. http:get:: /design/script/toggle/<stype>
+    .. :quickref: Workflow Design; update the UI state for the design canvas.
 
-    :status 200: and then redirects to :http:get:`/design/script`
-
-    """
-    script = utils.get_script_file()
-    script.editing_type = stype
-    utils.post_script_file(script)
-    return redirect(url_for('design.experiment_builder'))
+    .. http:patch:: /draft/ui-state
 
+    Update the UI state for the design canvas, including showing code overlays, setting editing types,
+    and handling deck selection.
 
-@design.route("/updateList", methods=['POST'])
-@login_required
-def update_list():
-    order = request.form['order']
-    script = utils.get_script_file()
-    script.currently_editing_order = order.split(",", len(script.currently_editing_script))
-    script.sort_actions()
-    exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
-    utils.post_script_file(script)
-    session['python_code'] = exec_string
-
-    return jsonify({'success': True})
+    :form show_code: Whether to show the code overlay (true/false).
+    :form editing_type: The type of editing to set (prep, script, cleanup).
+    :form autofill: Whether to enable autofill for the instrument panel (true/false).
+    :form deck_name: The name of the deck to select.
 
+    :status 200: Updates the UI state and returns a success message.
+    """
+    data = request.get_json()
 
-@design.route("/toggle_show_code", methods=["POST"])
-def toggle_show_code():
-    session["show_code"] = not session.get("show_code", False)
-    return redirect(request.referrer or url_for("design.experiment_builder"))
+    if "show_code" in data:
+        session["show_code"] = bool(data["show_code"])
+        return jsonify({"success": True})
+    if "editing_type" in data:
+        stype = data.get("editing_type")
 
+        script = utils.get_script_file()
+        script.editing_type = stype
+        utils.post_script_file(script)
 
-# --------------------handle all the import/export and download/upload--------------------------
-@design.route("/design/clear")
+        # Re-render only the part of the page you want to update
+        design_buttons = {stype: create_action_button(script, stype) for stype in script.stypes}
+        rendered_html = render_template("components/canvas.html", script=script, buttons_dict=design_buttons)
+        return jsonify({"html": rendered_html})
+
+    if "autofill" in data:
+        script = utils.get_script_file()
+        instrument = data.get("instrument", '')
+        autofill = data.get("autofill", False)
+        session['autofill'] = autofill
+        _, forms = _create_forms(instrument, script, autofill)
+        rendered_html = render_template("components/actions_panel.html", forms=forms, script=script, instrument=instrument)
+        return jsonify({"html": rendered_html})
+
+    if "deck_name" in data:
+        pkl_name = data.get('deck_name', "")
+        script = utils.get_script_file()
+        session['pseudo_deck'] = pkl_name
+        deck_list = utils.available_pseudo_deck(current_app.config["DUMMY_DECK"])
+
+        if script.deck is None or script.isEmpty():
+            script.deck = pkl_name.split('.')[0]
+            utils.post_script_file(script)
+        elif script.deck and not script.deck == pkl_name.split('.')[0]:
+            flash(f"Choose the deck with name {script.deck}")
+        pseudo_deck_path = os.path.join(current_app.config["DUMMY_DECK"], pkl_name)
+        pseudo_deck = utils.load_deck(pseudo_deck_path)
+        deck_variables = list(pseudo_deck.keys()) if pseudo_deck else []
+        deck_variables.remove("deck_name") if len(deck_variables) > 0 else deck_variables
+        html = render_template("components/sidebar.html", history=deck_list,
+                               defined_variables=deck_variables, local_variables = global_config.defined_variables,
+                               block_variables=global_config.building_blocks)
+        return jsonify({"html": html})
+    return jsonify({"error": "Invalid request"}), 400
+
+
+# @design.route("/draft/steps/order", methods=['POST'])
+# @login_required
+# def update_list():
+#     """
+#     .. :quickref: Workflow Design Steps; update the order of steps in the design canvas when reordering steps.
+#
+#     .. http:post:: /draft/steps/order
+#
+#     Update the order of steps in the design canvas when reordering steps.
+#
+#     :form order: A comma-separated string representing the new order of steps.
+#     :status 200: Successfully updated the order of steps.
+#     """
+#     order = request.form['order']
+#     script = utils.get_script_file()
+#     script.currently_editing_order = order.split(",", len(script.currently_editing_script))
+#     script.sort_actions()
+#     exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
+#     utils.post_script_file(script)
+#     session['python_code'] = exec_string
+#
+#     return jsonify({'success': True})
+
+
+
+@design.route("/draft", methods=['DELETE'])
 @login_required
-def clear():
+def clear_draft():
     """
     .. :quickref: Workflow Design; clear the design canvas.
 
-    .. http:get:: /design/clear
+    .. http:delete:: /draft
 
-    :form prompt: user's prompt
-    :status 200: clear canvas and then redirects to :http:get:`/design/script`
+    :status 200: clear canvas
     """
     deck = global_config.deck
-    pseudo_name = session.get("pseudo_deck", "")
     if deck:
         deck_name = os.path.splitext(os.path.basename(deck.__file__))[
             0] if deck.__name__ == "__main__" else deck.__name__
-    elif pseudo_name:
-        deck_name = pseudo_name
     else:
-        deck_name = ''
-    script = Script(deck=deck_name, author=session.get('username'))
+        deck_name = session.get("pseudo_deck", "")
+    script = Script(deck=deck_name, author=current_user.get_id())
     utils.post_script_file(script)
-    return redirect(url_for("design.experiment_builder"))
-
-
-@design.route("/design/import/pseudo", methods=['POST'])
-@login_required
-def import_pseudo():
-    """
-    .. :quickref: Workflow Design; Import pseudo deck from deck history
+    exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
+    session['python_code'] = exec_string
+    return jsonify({'success': True})
 
-    .. http:post:: /design/import/pseudo
 
-    :form pkl_name: pseudo deck name
-    :status 302: load pseudo deck and then redirects to :http:get:`/design/script`
-    """
-    pkl_name = request.form.get('pkl_name')
-    script = utils.get_script_file()
-    session['pseudo_deck'] = pkl_name
 
-    if script.deck is None or script.isEmpty():
-        script.deck = pkl_name.split('.')[0]
-        utils.post_script_file(script)
-    elif script.deck and not script.deck == pkl_name.split('.')[0]:
-        flash(f"Choose the deck with name {script.deck}")
-    return redirect(url_for("design.experiment_builder"))
 
 
-@design.route('/design/uploads', methods=['POST'])
-@login_required
-def upload():
+@design.route("/draft/submit_python", methods=["POST"])
+def submit_script():
     """
-    .. :quickref: Workflow Execution; upload a workflow config file (.CSV)
+    .. :quickref: Workflow Design; convert Python to workflow script
 
-    .. http:post:: /design/uploads
-
-    :form file: workflow CSV config file
-    :status 302: save csv file and then redirects to :http:get:`/design/campaign`
-    """
-    if request.method == "POST":
-        f = request.files['file']
-        if 'file' not in request.files:
-            flash('No file part')
-        if f.filename.split('.')[-1] == "csv":
-            filename = secure_filename(f.filename)
-            f.save(os.path.join(current_app.config['CSV_FOLDER'], filename))
-            session['config_file'] = filename
-            return redirect(url_for("design.experiment_run"))
-        else:
-            flash("Config file is in csv format")
-            return redirect(url_for("design.experiment_run"))
+    .. http:post:: /draft/submit_python
 
+    Convert a Python script to a workflow script and save it in the database.
 
-@design.route('/design/workflow/download/<filename>')
-@login_required
-def download_results(filename):
+    :form workflow_name: workflow name
+    :form script: main script
+    :form prep: prep script
+    :form cleanup: post script
+    :status 200: clear canvas
     """
-    .. :quickref: Workflow Design; download a workflow data file
-
-    .. http:get:: /design/workflow/download/<filename>
+    deck = global_config.deck
+    deck_name = os.path.splitext(os.path.basename(deck.__file__))[0] if deck.__name__ == "__main__" else deck.__name__
+    script = Script(author=current_user.get_id(), deck=deck_name)
+    script_collection = request.get_json()
+    workflow_name = script_collection.pop("workflow_name")
+    script.python_script = script_collection
+    # todo check script format
+    script.name = workflow_name
+    result = {}
+    for stype, py_str in script_collection.items():
+        try:
+            card = convert_to_cards(py_str)
+            script.script_dict[stype] = card
+            result[stype] = "success"
+        except Exception as e:
+            result[stype] = f"failed to transcript, but function can still run. error: {str(e)}"
+    utils.post_script_file(script)
+    status = publish()
+    return jsonify({"script": result, "db": status}), 200
 
-    """
-    filepath = os.path.join(current_app.config["DATA_FOLDER"], filename)
-    return send_file(os.path.abspath(filepath), as_attachment=True)
 
 
-@design.route('/design/load_json', methods=['POST'])
+@design.post("/draft/instruments/<string:instrument>")
 @login_required
-def load_json():
+def methods_handler(instrument: str = ''):
     """
-    .. :quickref: Workflow Design Ext; upload a workflow design file (.JSON)
+    .. :quickref: Workflow Design; handle methods of a specific instrument
 
-    .. http:post:: /load_json
+    .. http:post:: /draft/instruments/<string:instrument>
 
-    :form file: workflow design JSON file
-    :status 302: load pseudo deck and then redirects to :http:get:`/design/script`
-    """
-    if request.method == "POST":
-        f = request.files['file']
-        if 'file' not in request.files:
-            flash('No file part')
-        if f.filename.endswith("json"):
-            script_dict = json.load(f)
-            utils.post_script_file(script_dict, is_dict=True)
-        else:
-            flash("Script file need to be JSON file")
-    return redirect(url_for("design.experiment_builder"))
-
-
-@design.route('/design/script/download/<filetype>')
-@login_required
-def download(filetype):
-    """
-    .. :quickref: Workflow Design Ext; download a workflow design file
-
-    .. http:get:: /design/script/download/<filetype>
+    Add methods for a specific instrument in the workflow design.
 
+    :param instrument: The name of the instrument to handle methods for.
+    :type instrument: str
+    :status 200: Render the methods for the specified instrument.
     """
     script = utils.get_script_file()
-    run_name = script.name if script.name else "untitled"
-    if filetype == "configure":
-        filepath = os.path.join(current_app.config['SCRIPT_FOLDER'], f"{run_name}_config.csv")
-        with open(filepath, 'w', newline='') as f:
-            writer = csv.writer(f)
-            cfg, cfg_types = script.config("script")
-            writer.writerow(cfg)
-            writer.writerow(list(cfg_types.values()))
-    elif filetype == "script":
-        script.sort_actions()
-        json_object = json.dumps(script.as_dict())
-        filepath = os.path.join(current_app.config['SCRIPT_FOLDER'], f"{run_name}.json")
-        with open(filepath, "w") as outfile:
-            outfile.write(json_object)
-    elif filetype == "python":
-        filepath = os.path.join(current_app.config["SCRIPT_FOLDER"], f"{run_name}.py")
-    else:
-        return "Unsupported file type", 400
-    return send_file(os.path.abspath(filepath), as_attachment=True)
-
-
-@design.route("/design/step/edit/<uuid>", methods=['GET', 'POST'])
-@login_required
-def edit_action(uuid: str):
-    """
-    .. :quickref: Workflow Design; edit parameters of an action step on canvas
+    pseudo_deck_name = session.get('pseudo_deck', '')
+    pseudo_deck_path = os.path.join(current_app.config["DUMMY_DECK"], pseudo_deck_name)
+    off_line = current_app.config["OFF_LINE"]
+    pseudo_deck = utils.load_deck(pseudo_deck_path) if off_line and pseudo_deck_name else None
+    autofill = session.get('autofill', False)
 
-    .. http:get:: /design/step/edit/<uuid>
+    functions, forms = _create_forms(instrument, script, autofill, pseudo_deck)
 
-    Load parameter form of an action step
+    success = True
+    msg = ""
+    request.form
+    if "hidden_name" in request.form:
+        deck_snapshot = global_config.deck_snapshot
+        block_snapshot = global_config.building_blocks
+        method_name = request.form.get("hidden_name", None)
+        form = forms.get(method_name) if forms else None
+        insert_position = request.form.get("drop_target_id", None)
 
-    .. http:post:: /design/step/edit/<uuid>
+        if form:
+            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
+            if form.validate_on_submit():
+                function_name = kwargs.pop("hidden_name")
+                batch_action = kwargs.pop("batch_action", False)
+                save_data = kwargs.pop('return', '')
+                primitive_arg_types = utils.get_arg_type(kwargs, functions[function_name])
 
-    :param uuid: The step's uuid
-    :type uuid: str
+                # todo
+                # print(primitive_arg_types)
 
-    :form dynamic form: workflow step dynamic inputs
-    :status 302: save changes and then redirects to :http:get:`/design/script`
-    """
-    script = utils.get_script_file()
-    action = script.find_by_uuid(uuid)
-    session['edit_action'] = action
-
-    if request.method == "POST" and action is not None:
-        forms = create_form_from_action(action, script=script)
-        if "back" not in request.form:
-            kwargs = {field.name: field.data for field in forms if field.name != 'csrf_token'}
-            # print(kwargs)
-            if forms and forms.validate_on_submit():
-                save_as = kwargs.pop('return', '')
+                script.eval_list(kwargs, primitive_arg_types)
                 kwargs = script.validate_variables(kwargs)
-            # try:
-                script.update_by_uuid(uuid=uuid, args=kwargs, output=save_as)
-            # except Exception as e:
+                coroutine = False
+                if instrument.startswith("deck") and deck_snapshot:
+                    coroutine = deck_snapshot[instrument][function_name].get("coroutine", False)
+                elif instrument.startswith("blocks") and block_snapshot:
+                    coroutine = block_snapshot[instrument][function_name].get("coroutine", False)
+                action = {"instrument": instrument, "action": function_name,
+                          "args": kwargs,
+                          "return": save_data,
+                          'arg_types': primitive_arg_types,
+                          "coroutine": coroutine,
+                          "batch_action": batch_action,
+                          }
+                script.add_action(action=action, insert_position=insert_position)
             else:
-                flash(forms.errors)
-        session.pop('edit_action')
-    return redirect(url_for('design.experiment_builder'))
-
-
-@design.route("/design/step/delete/<id>")
-@login_required
-def delete_action(id: int):
-    """
-    .. :quickref: Workflow Design; delete an action step on canvas
-
-    .. http:get:: /design/step/delete/<id>
-
-    :param id: The step number id
-    :type id: int
-
-    :status 302: save changes and then redirects to :http:get:`/design/script`
-    """
-    back = request.referrer
-    script = utils.get_script_file()
-    script.delete_action(id)
+                msg = [f"{field}: {', '.join(messages)}" for field, messages in form.errors.items()]
+                success = False
+    elif "builtin_name" in request.form:
+        function_name = request.form.get("builtin_name")
+        form = forms.get(function_name) if forms else None
+        insert_position = request.form.get("drop_target_id", None)
+        if form:
+            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
+            if form.validate_on_submit():
+                logic_type = kwargs.pop('builtin_name')
+                if 'variable' in kwargs:
+                    try:
+                        script.add_variable(insert_position=insert_position, **kwargs)
+                    except ValueError:
+                        success = False
+                        msg = [f"{field}: {', '.join(messages)}" for field, messages in form.errors.items()]
+                else:
+                    script.add_logic_action(logic_type=logic_type, insert_position=insert_position, **kwargs)
+            else:
+                success = False
+                msg = [f"{field}: {', '.join(messages)}" for field, messages in form.errors.items()]
+    elif "workflow_name" in request.form:
+        workflow_name = request.form.get("workflow_name")
+        form = forms.get(workflow_name) if forms else None
+        insert_position = request.form.get("drop_target_id", None)
+        if form:
+            kwargs = {field.name: field.data for field in form if field.name != 'csrf_token'}
+            if form.validate_on_submit():
+                save_data = kwargs.pop('return', '')
+                primitive_arg_types = utils.get_arg_type(kwargs, functions[workflow_name])
+                script.eval_list(kwargs, primitive_arg_types)
+                kwargs = script.validate_variables(kwargs)
+                action = {"instrument": instrument, "action": workflow_name,
+                          "args": kwargs,
+                          "return": save_data,
+                          'arg_types': primitive_arg_types}
+                script.add_action(action=action, insert_position=insert_position)
+                script.add_workflow(**kwargs, insert_position=insert_position)
+            else:
+                success = False
+                msg = [f"{field}: {', '.join(messages)}" for field, messages in form.errors.items()]
     utils.post_script_file(script)
-    return redirect(back)
+    #TODO
+    try:
+        exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
+    except Exception as e:
+        exec_string = {}
+        msg = f"Compilation failed: {str(e)}"
+    # exec_string = script.compile(current_app.config['SCRIPT_FOLDER'])
+    session['python_code'] = exec_string
+    design_buttons = {stype: create_action_button(script, stype) for stype in script.stypes}
+    html = render_template("components/canvas_main.html", script=script, buttons_dict=design_buttons)
+    return jsonify({"html": html, "success": success, "error": msg})
 
 
-@design.route("/design/step/duplicate/<id>")
+@design.get("/draft/instruments", strict_slashes=False)
+@design.get("/draft/instruments/<string:instrument>")
 @login_required
-def duplicate_action(id: int):
+def get_operation_sidebar(instrument: str = ''):
     """
-    .. :quickref: Workflow Design; duplicate an action step on canvas
+    .. :quickref: Workflow Design; handle methods of a specific instrument
 
-    .. http:get:: /design/step/duplicate/<id>
+    .. http:get:: /draft/instruments/<string:instrument>
 
-    :param id: The step number id
-    :type id: int
+    :param instrument: The name of the instrument to handle methods for.
+    :type instrument: str
 
-    :status 302: save changes and then redirects to :http:get:`/design/script`
+    :status 200: Render the methods for the specified instrument.
     """
-    back = request.referrer
     script = utils.get_script_file()
-    script.duplicate_action(id)
-    utils.post_script_file(script)
-    return redirect(back)
-
-
-# ---- HTTP API Endpoints ----
-
-@design.route("/api/runner/status", methods=["GET"])
-def runner_status():
-    """
-    .. :quickref: Workflow Design; get the execution status
-
-    .. http:get:: /api/runner/status
-
-    :status 200: status
-    """
-    runner_busy = global_config.runner_lock.locked()
-    status = {"busy": runner_busy}
-    task_status = global_config.runner_status
-    current_step = {}
-    # print(task_status)
-    if task_status is not None:
-        task_type = task_status["type"]
-        task_id = task_status["id"]
-        if task_type == "task":
-            step = SingleStep.query.get(task_id)
-            current_step = step.as_dict()
-        if task_type == "workflow":
-            workflow = WorkflowRun.query.get(task_id)
-            if workflow is not None:
-                latest_step = WorkflowStep.query.filter_by(workflow_id=workflow.id).order_by(WorkflowStep.start_time.desc()).first()
-                if latest_step is not None:
-                    current_step = latest_step.as_dict()
-                status["workflow_status"] = {"workflow_info": workflow.as_dict(), "runner_status": runner.get_status()}
-    status["current_task"] = current_step
-    return jsonify(status), 200
-
-
-
-@design.route("/api/runner/abort_pending", methods=["POST"])
-def api_abort_pending():
-    """
-    .. :quickref: Workflow Design; abort pending action(s) during execution
-
-    .. http:get:: /api/runner/abort_pending
-
-    :status 200: {"status": "ok"}
-    """
-    abort_pending()
-    return jsonify({"status": "ok"}), 200
-
-@design.route("/api/runner/abort_current", methods=["POST"])
-def api_abort_current():
-    """
-    .. :quickref: Workflow Design; abort right after current action during execution
-
-    .. http:get:: /api/runner/abort_current
-
-    :status 200: {"status": "ok"}
-    """
-    abort_current()
-    return jsonify({"status": "ok"}), 200
-
-@design.route("/api/runner/pause", methods=["POST"])
-def api_pause():
-    """
-    .. :quickref: Workflow Design; pause during execution
-
-    .. http:get:: /api/runner/pause
-
-    :status 200: {"status": "ok"}
-    """
-    msg = pause()
-    return jsonify({"status": "ok", "pause_status": msg}), 200
-
-@design.route("/api/runner/retry", methods=["POST"])
-def api_retry():
-    """
-    .. :quickref: Workflow Design; retry when error occur during execution
-
-    .. http:get:: /api/runner/retry
-
-    :status 200: {"status": "ok"}
-    """
-    retry()
-    return jsonify({"status": "ok, retrying failed step"}), 200
+    pseudo_deck_name = session.get('pseudo_deck', '')
+    pseudo_deck_path = os.path.join(current_app.config["DUMMY_DECK"], pseudo_deck_name)
+    off_line = current_app.config["OFF_LINE"]
+    pseudo_deck = utils.load_deck(pseudo_deck_path) if off_line and pseudo_deck_name else None
+    autofill = session.get('autofill', False)
 
+    functions, forms = _create_forms(instrument, script, autofill, pseudo_deck)
 
-@design.route("/api/design/submit", methods=["POST"])
-def submit_script():
-    """
-    .. :quickref: Workflow Design; submit script
+    if instrument:
+        html = render_template("components/sidebar.html", forms=forms, instrument=instrument, script=script)
+    else:
+        pseudo_deck_name = session.get('pseudo_deck', '')
+        pseudo_deck_path = os.path.join(current_app.config["DUMMY_DECK"], pseudo_deck_name)
+        off_line = current_app.config["OFF_LINE"]
+        pseudo_deck = utils.load_deck(pseudo_deck_path) if off_line and pseudo_deck_name else None
+        if off_line and pseudo_deck is None:
+            flash("Choose available deck below.")
+        deck_list = utils.available_pseudo_deck(current_app.config["DUMMY_DECK"])
+        if not off_line:
+            deck_variables = list(global_config.deck_snapshot.keys())
+        else:
+            deck_variables = list(pseudo_deck.keys()) if pseudo_deck else []
+            deck_variables.remove("deck_name") if len(deck_variables) > 0 else deck_variables
+        # edit_action_info = session.get("edit_action")
+        html = render_template("components/sidebar.html", off_line=off_line, history=deck_list,
+                               defined_variables=deck_variables,
+                               local_variables=global_config.defined_variables,
+                               block_variables=global_config.building_blocks,
+                               )
+    return jsonify({"html": html})
 
-    .. http:get:: /api/design/submit
 
-    :status 200: {"status": "ok"}
-    """
-    deck = global_config.deck
-    deck_name = os.path.splitext(os.path.basename(deck.__file__))[0] if deck.__name__ == "__main__" else deck.__name__
-    script = Script(author=session.get('user'), deck=deck_name)
-    script_collection = request.get_json()
-    script.python_script = script_collection
-    # todo check script format
-    utils.post_script_file(script)
-    return jsonify({"status": "ok"}), 200