oocana-python-executor 0.15.0__py3-none-any.whl

oocana_python_executor-0.15.0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: oocana-python-executor
+Version: 0.15.0
+Summary: a client subscribe mqtt topic to execute oocana's block
+Author-Email: l1shen <lishen1635@gmail.com>, yleaf <11785335+leavesster@users.noreply.github.com>
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: oocana
+

oocana_python_executor-0.15.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+oocana_python_executor-0.15.0.dist-info/METADATA,sha256=yWmQbi6QROwN3iesVi_D4BtL8We8twY6Idk_owqDWY8,289
+oocana_python_executor-0.15.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+oocana_python_executor-0.15.0.dist-info/entry_points.txt,sha256=n5hnDKR-LHdMNSqVy89zHBqyOMcRqva6KJrajUFfBnk,82
+python_executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+python_executor/block.py,sha256=DhKQwAkFfNHQvM0Y_uDxA6XejviEp2h3-Ead_uoG7q0,6209
+python_executor/context.py,sha256=v_STYkHDNKHVwPiwhiE23IfjzGxRncVTrrZqsc6q76Y,2549
+python_executor/data.py,sha256=KF-LNS1fZkjRSTSt8jrVTgZV712y6rEPF5Qi2pqutdk,139
+python_executor/executor.py,sha256=IzSKRkMjR9qyh-M15h4OiC5xtzbSDRR1529GQXKhSOY,10023
+python_executor/hook.py,sha256=lqWkK2I4I8H8AjYOv0YV5hk8X1ZdECCyueW4gU7JL48,1523
+python_executor/logger.py,sha256=kzY_0QT3M72JP1WYyk3p9LVo-iEd6DZjwpwoswb6v8U,418
+python_executor/matplot/matplotlib_oomol/__init__.py,sha256=_5Y6euJwP32h4I8WMtlZ-tJzGOqFk9v6CItmWSQyJek,37
+python_executor/matplot/matplotlib_oomol/oomol.py,sha256=WIdaICAYrJ0nw67hdPHxAOfOQjgvHmeOM-WXwCb2Rr8,1057
+python_executor/matplot/oomol_matplot_helper.py,sha256=lqNM7jl4pjzB0vo_ubYfXn1JSuZf3OKDrGXxhdnPDig,2694
+python_executor/secret.py,sha256=-dAnbNKv9XK-zfxoMulIKWDGt4NHAXJ45cCLT9iBiKQ,5984
+python_executor/service.py,sha256=XcYz7B7Mc_x5LpEw9b81gDmcNvUaZh4UIngDeOMUVwA,9209
+python_executor/topic.py,sha256=YAMD5eZwjYXQYJrYLdCBh-xt2G3jRI1eEed-0K9E6XA,2204
+python_executor/utils.py,sha256=3liF5N94KDbj41wbqiD9Pt0SNtLi-uP8jXMyyUheYQ0,892
+oocana_python_executor-0.15.0.dist-info/RECORD,,

oocana_python_executor-0.15.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: pdm-backend (2.4.3)
+Root-Is-Purelib: true
+Tag: py3-none-any

oocana_python_executor-0.15.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+python-executor = python_executor.executor:main
+
+[gui_scripts]
+

python_executor/block.py

@@ -0,0 +1,189 @@
+from oocana import Context, Mainframe
+from dataclasses import dataclass
+from typing import Optional, TypedDict
+import inspect
+import traceback
+import logging
+from .data import store, vars, EXECUTOR_NAME
+from .context import createContext
+from .hook import ExitFunctionException
+import os
+import sys
+import importlib
+import importlib.util
+import threading
+
+
+class ExecutorOptionsDict(TypedDict):
+    function: Optional[str]
+    entry: Optional[str]
+    source: Optional[str]
+
+# entry 与 source 是二选一的存在
+class ExecutorDict(TypedDict):
+    options: Optional[ExecutorOptionsDict]
+
+tmp_files = set()
+
+@dataclass
+class ExecutePayload:
+    session_id: str
+    job_id: str
+    dir: str
+    executor: ExecutorDict
+    outputs: Optional[dict] = None
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            self.session_id = args[0]
+            self.job_id = args[1]
+            self.executor = args[2]
+            self.dir = args[3]
+            self.outputs = args[4]
+        if kwargs:
+            for key, value in kwargs.items():
+                setattr(self, key, value)
+
+lock = threading.Lock()
+
+def load_module(file_path: str, source_dir=None):
+
+    if (os.path.isabs(file_path)):
+        file_abs_path = file_path
+    else:
+        dirname = source_dir if source_dir else os.getcwd()
+        file_abs_path = os.path.abspath(os.path.join(dirname, file_path))
+    with lock:
+        if file_abs_path in sys.modules:
+            return sys.modules[file_abs_path]
+
+        module_dir = os.path.dirname(file_abs_path)
+        sys.path.insert(0, module_dir)
+
+        file_spec = importlib.util.spec_from_file_location(file_abs_path, file_abs_path)
+        module = importlib.util.module_from_spec(file_spec)  # type: ignore
+        sys.modules[file_abs_path] = module
+
+        file_spec.loader.exec_module(module)  # type: ignore
+    return module
+
+
+def output_return_object(obj, context: Context):
+    if obj is None:
+        context.done()
+    elif obj is context.keepAlive:
+        pass
+    elif isinstance(obj, dict):
+        for k, v in obj.items():
+            context.output(k, v)
+        context.done()
+    else:
+        context.done(f"return object needs to be a dictionary, but get type: {type(obj)}")
+
+logger = logging.getLogger(EXECUTOR_NAME)
+
+async def run_block(message, mainframe: Mainframe, session_dir: str):
+
+    logger.info(f"block {message.get('job_id')} start")
+    try:
+        payload = ExecutePayload(**message)
+        context = createContext(mainframe, payload.session_id, payload.job_id, store, payload.outputs, session_dir)
+    except Exception:
+        traceback_str = traceback.format_exc()
+        # rust 那边会保证传过来的 message 一定是符合格式的，所以这里不应该出现异常。这里主要是防止 rust 修改错误。
+        mainframe.send(
+        {
+            "job_id": message["job_id"],
+            "session_id": message["session_id"],
+        },
+        {
+            "type": "BlockFinished",
+            "job_id": message["job_id"], 
+            "error": traceback_str
+        })
+        return
+
+    vars.set(context)
+
+    load_dir = payload.dir
+
+    options = payload.executor.get("options")
+
+    node_id = context.node_id
+
+    file_path = options["entry"] if options is not None and options.get("entry") is not None else 'index.py'
+
+    source = options.get("source") if options is not None else None
+    if source is not None:
+        if not os.path.exists(load_dir):
+            os.makedirs(load_dir)
+        
+        dir_path = os.path.join(load_dir, ".scriptlets")
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+        tmp_py = os.path.join(dir_path, f"{node_id}.py")
+        # 记录临时文件，但是现在不再清理
+        tmp_files.add(tmp_py)
+
+        with open(tmp_py, "w") as f:
+            f.write(source)
+        file_path = tmp_py
+
+    try:
+        # TODO: 这里的异常处理，应该跟详细一些，提供语法错误提示。
+        index_module = load_module(file_path, load_dir) # type: ignore
+    except Exception:
+        traceback_str = traceback.format_exc()
+        context.done(traceback_str)
+        return
+    function_name: str = options.get("function") if payload.executor is not None and options.get("function") is not None else 'main' # type: ignore
+    fn = index_module.__dict__.get(function_name)
+
+    if fn is None:
+        context.done(f"function {function_name} not found in {file_path}")
+        return
+    if not callable(fn):
+        context.done(f"{function_name} is not a function in {file_path}")
+        return
+
+    try:
+        signature = inspect.signature(fn)
+        params_count = len(signature.parameters)
+        result = None
+        traceback_str = None
+
+        try:
+            if inspect.iscoroutinefunction(fn):
+                if params_count == 0:
+                    result = await fn()
+                elif params_count == 1:
+                    only_context_param = list(signature.parameters.values())[0].annotation is Context
+                    result = await fn(context) if only_context_param else await fn(context.inputs)
+                else:
+                    result = await fn(context.inputs, context)
+            else:
+                if params_count == 0:
+                    result = fn()
+                elif params_count == 1:
+                    only_context_param = list(signature.parameters.values())[0].annotation is Context
+                    result = fn(context) if only_context_param else fn(context.inputs)
+                else:
+                    result = fn(context.inputs, context)
+        except ExitFunctionException as e:
+            if e.args[0] is not None:
+                context.done("block call exit with message: " + str(e.args[0]))
+            else:
+                context.done()
+        except Exception:
+            traceback_str = traceback.format_exc()
+
+        if traceback_str is not None:
+            context.done(traceback_str)
+        else:
+            output_return_object(result, context)
+    except Exception:
+        traceback_str = traceback.format_exc()
+        context.done(traceback_str)
+    finally:
+        logger.info(f"block {message.get('job_id')} done")
+

python_executor/context.py

@@ -0,0 +1,68 @@
+import logging
+from oocana import Mainframe, Context, StoreKey, BlockInfo, BinValueDict, VarValueDict, InputHandleDef, is_bin_value, is_var_value
+from typing import Dict
+from .secret import replace_secret
+import os.path
+from .logger import ContextHandler
+from .data import EXECUTOR_NAME
+
+logger = logging.getLogger(EXECUTOR_NAME)
+
+def createContext(
+    mainframe: Mainframe, session_id: str, job_id: str, store, output, session_dir: str
+) -> Context:
+
+    node_props = mainframe.notify_block_ready(session_id, job_id)
+
+    inputs_def: Dict[str, Dict] | None = node_props.get("inputs_def")
+    inputs = node_props.get("inputs")
+
+    if inputs_def is not None and inputs is not None:
+
+        inputs_def_handles: Dict[str, InputHandleDef] = {}
+        for k, v in inputs_def.items():
+            inputs_def_handles[k] = InputHandleDef(**v)
+
+        inputs = replace_secret(inputs, inputs_def_handles, node_props.get("inputs_def_patch"))
+
+        for k, v in inputs.items():
+            input_def = inputs_def_handles.get(k)
+            if input_def is None:
+                continue
+            if is_var_value(v):
+                wrap_var: VarValueDict = v
+                try:
+                    ref = StoreKey(**wrap_var["value"])
+                except:  # noqa: E722
+                    logger.warning(f"not valid object ref: {wrap_var}")
+                    continue
+                if ref in store:
+                    inputs[k] = store.get(ref)
+                else:
+                    logger.error(f"object {ref} not found in store")
+            elif is_bin_value(v):
+                wrap_bin: BinValueDict = v
+                path = wrap_bin["value"]
+                if isinstance(path, str):
+                    # check file path v is exist
+                    if not os.path.exists(path):
+                        logger.error(f"file {path} for oomol/bin is not found")
+                        continue
+
+                    with open(path, "rb") as f:
+                        inputs[k] = f.read()
+                else:
+                    logger.error(f"not valid bin handle: {v}")
+
+    if inputs is None:
+        inputs = {}
+    
+    blockInfo = BlockInfo(**node_props)
+
+    ctx = Context(inputs, blockInfo, mainframe, store, output, session_dir)
+    # 跟 executor 日志分开，避免有的库在 logger 里面使用 print，导致 hook 出现递归调用。
+    block_logger = logging.getLogger(f"block {job_id}")
+    ctx_handler = ContextHandler(ctx)
+    block_logger.addHandler(ctx_handler)
+    ctx._logger = logger
+    return ctx

python_executor/data.py

@@ -0,0 +1,5 @@
+from contextvars import ContextVar
+from oocana import Context, EXECUTOR_NAME
+
+vars: ContextVar[Context] = ContextVar('context')
+store = {}

python_executor/executor.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+
+import asyncio
+import os
+import queue
+import sys
+import logging
+from . import hook
+from oocana import Mainframe, ServiceExecutePayload
+from .utils import run_in_new_thread, run_async_code, oocana_dir
+from .block import run_block
+from oocana import EXECUTOR_NAME
+from .matplot.oomol_matplot_helper import import_helper, add_matplot_module
+from typing import Literal
+from .topic import prepare_report_topic, service_config_topic, run_action_topic, ServiceTopicParams, ReportStatusPayload, exit_report_topic, status_report_topic
+
+logger = logging.getLogger(EXECUTOR_NAME)
+service_store: dict[str, Literal["launching", "running"]] = {}
+job_set = set()
+
+# 日志目录 ~/.oocana/sessions/{session_id}
+# executor 的日志都会记录在 [python-executor-{suffix}.log | python-executor.log]
+# 全局 logger 会记录在 python-{suffix}.log | python.log
+def config_logger(session_id: str, suffix: str | None, output: Literal["console", "file"]):
+
+
+    format = '%(asctime)s - %(levelname)s - {%(pathname)s:%(lineno)d} - %(message)s'
+    fmt = logging.Formatter(format)
+    logger.setLevel(logging.DEBUG)
+    if output == "file":
+        executor_dir = os.path.join(oocana_dir(), "sessions", session_id)
+        logger_file = os.path.join(executor_dir, f"python-executor-{suffix}.log") if suffix is not None else os.path.join(executor_dir, "python-executor.log")
+
+        if not os.path.exists(logger_file):
+            os.makedirs(os.path.dirname(logger_file), exist_ok=True)
+
+        print(f"setup logging in file {logger_file}")
+        h = logging.FileHandler(logger_file)
+
+        global_logger_file = os.path.join(executor_dir, f"python-{suffix}.log") if suffix is not None else os.path.join(executor_dir, "python.log")
+        logging.basicConfig(filename=global_logger_file, level=logging.DEBUG, format=format)
+    else:
+        logging.basicConfig(level=logging.DEBUG, format=format)
+        h = logging.StreamHandler(sys.stdout)
+
+    h.setFormatter(fmt)
+    logger.addHandler(h)
+    # 跟全局日志分开。避免有的库在全局 logger 里面使用了 print 等 API，导致 hook 出现递归调用
+    logger.propagate = False
+
+
+async def run_executor(address: str, session_id: str, package: str | None, session_dir: str, suffix: str | None = None):
+
+    if suffix is not None:
+        mainframe = Mainframe(address, f"python-executor-{suffix}", logger)
+    else:
+        mainframe = Mainframe(address, f"python-executor-{session_id}", logger)
+
+    mainframe.connect()
+
+    print(f"connecting to broker {address} success")
+    sys.stdout.flush()
+
+    logger.info("executor start") if package is None else logger.info(f"executor start for package {package}")
+
+    add_matplot_module()
+    import_helper(logger)
+
+    # add package to sys.path
+    if package is not None:
+        sys.path.append(package)
+    elif os.path.exists("/app/workspace"):
+        sys.path.append("/app/workspace")
+
+
+    def not_current_session(message):
+        return message.get("session_id") != session_id
+    
+    def not_current_package(message):
+        return message.get("package") != package
+
+    # 目前的 mqtt 库，在 subscribe 回调里 publish 消息会导致死锁无法工作，参考 https://github.com/eclipse/paho.mqtt.python/issues/527 或者 https://stackoverflow.com/a/36964192/4770006
+    # 通过这种方式来绕过，所有需要 callback 后 publish message 的情况，都需要使用 future 类似方式来绕过。
+    fs = queue.Queue()
+    loop = asyncio.get_event_loop()
+
+    def execute_block(message):
+        if not_current_session(message):
+            return
+
+        if not_current_package(message):
+            return
+        
+        # https://github.com/oomol/oocana-rust/issues/310 临时解决方案
+        job_id = message.get("job_id")
+        if job_id in job_set:
+            logger.warning(f"job {job_id} already running, ignore")
+            return
+        job_set.add(job_id)
+
+        nonlocal fs
+        f = loop.create_future()
+        fs.put(f)
+        f.set_result(message)
+    
+    def execute_service_block(message):
+        if not_current_session(message):
+            return
+        
+        if not_current_package(message):
+            return
+
+        nonlocal fs
+        f = loop.create_future()
+        fs.put(f)
+        f.set_result(message)
+
+    def service_exit(message: ReportStatusPayload):
+        service_hash = message.get("service_hash")
+        if service_hash in service_store:
+            del service_store[service_hash]
+
+    def service_status(message: ReportStatusPayload):
+        service_hash = message.get("service_hash")
+        if service_hash in service_store:
+            service_store[service_hash] = "running"
+
+    def report_message(message):
+        type = message.get("type")
+        if type == "SessionFinished":
+            if not_current_session(message):
+                return
+            logger.info(f"session {session_id} finished, exit executor")
+            mainframe.disconnect() # TODO: 即使调用 disconnect，在 broker 上也无法看不到主动断开的信息，有时间再调查。
+            if os.getenv("IS_FORKED"): # fork 进程无法直接使用 sys.exit 退出
+                os._exit(0)
+            else:
+                hook.original_exit(0)
+        
+
+    mainframe.subscribe(f"executor/{EXECUTOR_NAME}/run_block", execute_block)
+    mainframe.subscribe(f"executor/{EXECUTOR_NAME}/run_service_block", execute_service_block)
+    mainframe.subscribe('report', report_message)
+    mainframe.subscribe(exit_report_topic(), service_exit)
+    mainframe.subscribe(status_report_topic(), service_status)
+
+    mainframe.notify_executor_ready(session_id, EXECUTOR_NAME, package)
+
+    async def spawn_service(message: ServiceExecutePayload, service_hash: str):
+        logger.info(f"create new service {message.get('dir')}")
+        service_store[service_hash] = "launching"
+
+        parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+        is_global_service = message.get("service_executor").get("stop_at") in ["app_end", "never"]
+
+        if is_global_service:
+            process = await asyncio.create_subprocess_shell(
+                f"python -u -m python_executor.service --address {address}  --service-hash {service_hash} --session-dir {session_dir}",
+                cwd=parent_dir
+            )
+        else:
+            process = await asyncio.create_subprocess_shell(
+                f"python -u -m python_executor.service --address {address} --session-id {session_id}  --service-hash {service_hash} --session-dir {session_dir}",
+                cwd=parent_dir
+            )
+        params: ServiceTopicParams = {
+            "service_hash": service_hash,
+            "session_id": session_id
+        }
+
+        def send_service_config(params: ServiceTopicParams, message: ServiceExecutePayload):
+
+            async def run():
+                mainframe.publish(service_config_topic(params), message)
+                service_store[service_hash] = "running"
+            run_in_new_thread(run)
+
+        # FIXME: mqtt 不能在 subscribe 后立即 publish，需要修复。
+        mainframe.subscribe(prepare_report_topic(params), lambda _: send_service_config(params, message))
+
+        await process.wait()
+        logger.info(f"service {service_hash} exit")
+        del service_store[service_hash]
+    
+
+    def run_service_block(message: ServiceExecutePayload):
+        logger.info(f"service block {message.get('job_id')} start")
+        service_hash = message.get("service_hash")
+        params: ServiceTopicParams = {
+            "service_hash": service_hash,
+            "session_id": session_id
+        }
+        mainframe.publish(run_action_topic(params), message)
+
+    while True:
+        await asyncio.sleep(1)
+        if not fs.empty():
+            f = fs.get()
+            message = await f
+            if message.get("service_executor") is not None:
+                service_hash = message.get("service_hash")
+                status = service_store.get(service_hash)
+                if status is None:
+                    asyncio.create_task(spawn_service(message, service_hash))
+                elif status == "running":
+                    run_service_block(message)
+                elif status == "launching":
+                    logger.info(f"service {service_hash} is launching, set message back to fs to wait next time")
+                    fs.put(f)
+            else:
+                if not_current_session(message):
+                    continue
+                run_block_in_new_thread(message, mainframe, session_dir=session_dir)
+
+def run_block_in_new_thread(message, mainframe: Mainframe, session_dir: str):
+
+    async def run():
+        await run_block(message, mainframe, session_dir=session_dir)
+    run_in_new_thread(run)
+
+def main():
+
+    import argparse
+    parser = argparse.ArgumentParser(description="run executor with address, session-id, tmp-dir")
+    parser.add_argument("--session-id", help="executor subscribe session id", required=True)
+    parser.add_argument("--address", help="mqtt address", default="mqtt://127.0.0.1:47688")
+    parser.add_argument("--session-dir", help="a tmp dir for whole session", required=True)
+    parser.add_argument("--output", help="output log to console or file", default="file", choices=["console", "file"])
+    parser.add_argument("--package", help="package path, if set, executor will only run same package block", default=None)
+    parser.add_argument("--suffix", help="suffix for log file", default=None)
+
+    args = parser.parse_args()
+
+    address: str = args.address
+    session_id: str = str(args.session_id)
+    output: Literal["console", "file"] = args.output
+    package: str | None = args.package
+    suffix: str | None = args.suffix
+    session_dir: str = args.session_dir
+
+    config_logger(session_id, suffix, output)
+
+    run_async_code(run_executor(address=address, session_id=session_id, package=package, session_dir=session_dir, suffix=suffix))
+
+if __name__ == '__main__':
+    main()

python_executor/hook.py

@@ -0,0 +1,50 @@
+from sys import exit
+from builtins import exit as global_exit
+from typing import TypeAlias, Any
+import sys
+import builtins
+from .data import vars, EXECUTOR_NAME
+import logging
+
+logger = logging.getLogger(EXECUTOR_NAME)
+
+class ExitFunctionException(Exception):
+    pass
+
+original_exit = exit
+original_global_exit = global_exit
+original_print = print
+
+_ExitCode: TypeAlias = str | int | None
+
+def sys_exit(status: _ExitCode = None) -> None:
+    raise ExitFunctionException(status)
+
+def sys_global_exit(status: _ExitCode = None) -> None:
+    raise ExitFunctionException(status)
+
+def global_print(*values: object, sep: str | None = " ", end: str | None = "\n", file: Any | None = None, flush: bool = False) -> None:
+    
+    context = None  # 初始化 context 变量
+    try:
+        context = vars.get()
+    except LookupError:
+        # 这个 logger 不会上报到 root handle 中，所以即使 root logger 的 Handler 里面有 print 函数，也不会导致递归调用
+        logger.warning("print called outside of block")
+    except Exception as e:
+        logger.error(f"print error: {e}")
+
+    if context is not None:
+        try:
+            msg_sep = sep or " "
+            msg = msg_sep.join(map(str, values))
+            context.report_log(msg)
+        except Exception as e:
+            logger.error(f"transform print message to context log error: {e}")
+
+    original_print(*values, sep=sep, end=end, file=file, flush=flush)
+
+
+sys.exit = sys_exit
+builtins.exit = sys_global_exit
+builtins.print = global_print

python_executor/logger.py

@@ -0,0 +1,19 @@
+import logging
+from oocana import Context
+import weakref
+
+class ContextHandler(logging.Handler):
+
+    @property
+    def context(self):
+        return self._context
+
+    def __init__(self, context: Context):
+        super().__init__()
+        self._context = weakref.ref(context)
+
+    def emit(self, record):
+        msg = self.format(record)
+        ctx = self.context()
+        if ctx:
+            ctx.report_log(msg)

python_executor/matplot/matplotlib_oomol/__init__.py

@@ -0,0 +1 @@
+from .oomol import show, FigureCanvas

python_executor/matplot/matplotlib_oomol/oomol.py

@@ -0,0 +1,28 @@
+"""matplotlib.use('module://matplotlib_oomol'), remember to add this file to PYTHONPATH"""
+
+from matplotlib.backend_bases import Gcf # type: ignore
+from matplotlib.backends.backend_agg import FigureCanvasAgg # type: ignore
+from python_executor.data import vars
+
+FigureCanvas = FigureCanvasAgg
+
+def show(*args, **kwargs):
+    import sys
+    from io import BytesIO
+    from base64 import b64encode
+    if vars is not None:
+        context = vars.get()
+        images = []
+        for figmanager in Gcf.get_all_fig_managers():
+                buffer = BytesIO()
+                figmanager.canvas.figure.savefig(buffer, format='png')
+                buffer.seek(0)
+                png = buffer.getvalue()
+                buffer.close()
+                base64Data = b64encode(png).decode('utf-8')
+                url = f'data:image/png;base64,{base64Data}'
+                images.append(url)
+        if images:
+            context.preview({ "type": "image", "data": images })
+    else:
+        print('matplotlib_oomol: no sys.modules["oomol"]', file=sys.stderr)

python_executor/matplot/oomol_matplot_helper.py

@@ -0,0 +1,68 @@
+from python_executor.data import vars
+
+def add_matplot_module():
+    import sys
+    import os.path
+    dir = os.path.dirname(os.path.abspath(__file__))
+    sys.path.insert(0, dir)
+
+def import_helper(logger):
+    # matplotlib 的 use() 替换
+    try:
+        import matplotlib # type: ignore
+        matplotlib.use('module://matplotlib_oomol') # matplotlib_oomol.py 文件所在目录加入 PYTHONPATH
+    except:
+        logger.error("import matplotlib failed")
+
+    # matplotlib 主题替换
+    try:
+        import os
+        import matplotlib.pyplot as plt # type: ignore
+        plt.style.use("classic" if os.getenv("OOMOL_COLOR_SCHEME", "dark") == "light" else "dark_background")
+        plt.rcParams['font.sans-serif'] = ['Source Han Sans SC']
+    except:
+        pass
+
+    # plotly 的 show() 替换
+    try:
+        import os
+        import plotly.io as pio # type: ignore
+        from plotly.io import renderers # type: ignore
+        from plotly.io.base_renderers import ExternalRenderer # type: ignore
+
+        pio.templates.default = "plotly" if os.getenv("OOMOL_COLOR_SCHEME", "dark") == "light" else "plotly_dark"
+
+        class OomolRenderer(ExternalRenderer):
+            def render(self, fig_dict):
+                if vars:
+                    context = vars.get()
+
+                    import re
+                    from plotly.io import to_html # type: ignore
+                    from plotly.offline import get_plotlyjs_version # type: ignore
+
+                    cdn_ver = get_plotlyjs_version()
+                    cdn_url = f"https://cdn.jsdelivr.net/npm/plotly.js-dist-min@{cdn_ver}/plotly.min.js"
+
+                    html = to_html(
+                        fig_dict,
+                        include_plotlyjs=cdn_url,
+                        include_mathjax="cdn",
+                        full_html=True,
+                        default_width="100%",
+                        default_height="100%",
+                        validate=False,
+                    )
+
+                    color_scheme = os.getenv("OOMOL_COLOR_SCHEME", "dark")
+                    # The generated html has default body margin 8px in chrome, remove it.
+                    html = re.sub(r'<html[^>]*?>', r'\g<0><style>html { color-scheme: ' + color_scheme + '; height: 100%; align-content: center } ' +
+                        'body { overflow: hidden; margin: 0 }</style>', html, flags=re.I)
+                    context.preview({ "type": "html", "data": html })
+                else:
+                    logger.warning('plotly: no sys.modules["oomol"]')
+
+        renderers['oomol'] = OomolRenderer()
+        renderers.default = 'oomol'
+    except:
+        logger.warning("import plotly failed")

python_executor/secret.py

@@ -0,0 +1,146 @@
+from typing import Any, Dict
+from oocana import InputHandleDef, FieldSchema, ObjectFieldSchema, ArrayFieldSchema, SecretFieldSchema
+import logging
+import os
+import json
+import re
+from .data import EXECUTOR_NAME
+
+logger = logging.getLogger(EXECUTOR_NAME)
+SECRET_FILE =  os.path.expanduser("~") + "/app-config/oomol-secrets/secrets.json"
+
+# "${{OO_SECRET:type,name,key}}"，捕获组为 (type,name,key)
+SECRET_REGEX = r"\"\$\{\{OO_SECRET:([\S]+),([\S]+),([\S]+)\}\}\""
+
+def replace_secret(
+    value: Any,
+    root_def: Dict[str, InputHandleDef],
+    patch: Dict[str, Any] | None = None,
+) -> Any:
+    if not isinstance(value, dict):
+        return value
+    
+    try:
+        secretJson = json.load(open(SECRET_FILE))
+    except FileNotFoundError:
+        logger.warning(f"secret file {SECRET_FILE} not found")
+        secretJson = None
+    except json.JSONDecodeError:
+        logger.error(f"secret file {SECRET_FILE} is not a valid json file")
+        secretJson = None
+
+    string_value = json.dumps(value)
+    result = re.findall(SECRET_REGEX, string_value, re.MULTILINE)
+    if result:
+        if secretJson is None:
+            logger.error(f"secret file {SECRET_FILE} not found")
+        else:
+            for r in result:
+                secret = get_secret(f"{r[0]},{r[1]},{r[2]}", secretJson).replace('"', '\\"')
+                string_value = string_value.replace(f"${{{{OO_SECRET:{r[0]},{r[1]},{r[2]}}}}}", secret)
+        value = json.loads(string_value)
+    
+    for k, v in value.items():
+        input_def = root_def.get(k)
+        if input_def is None:
+            continue
+        # 如果是 None 就不替换，直接透传。只有 nullable 的情况下才会出现 None
+        if input_def.is_secret_handle() and v is not None:
+            value[k] = get_secret(v, secretJson)
+        # 为了保持功能聚焦，var 部分在 Context 那边重新迭代处理。var 只在根目录，同时重复迭代开销不大。如果要递归，还是要合并进来。
+        elif isinstance(input_def.json_schema, ObjectFieldSchema) or isinstance(input_def.json_schema, ArrayFieldSchema):
+            replace_sub_secret(v, input_def.json_schema, secretJson)
+
+    if patch is not None:
+        patch_def = patch # 避免下面的类型提示错误
+        for k, v in patch_def.items():
+            input_value = value.get(k)
+            if input_value is not None:
+                for patch in v:
+                    is_secret = patch_def.get("schema", {}).get("contentMediaType") == "oomol/secret"
+                    if not is_secret:
+                        continue
+                    
+                    path = patch_def.get("path")
+                    if path is None:
+                        # 整个替换，所以得是 value 而不是 input_value
+                        value[k] = get_secret(input_value, secretJson)
+                    elif isinstance(path, str) and path in input_value:
+                        input_value[path] = get_secret(input_value[path], secretJson)
+                    elif isinstance(path, int) and path < len(input_value):
+                        input_value[path] = get_secret(input_value[path], secretJson)
+                    elif isinstance(path, list):
+                        tmp = input_value
+                        for p in path[:-1]:
+                            tmp = tmp[p]
+                            if tmp is None:
+                                logger.error(f"invalid path: {path}")
+                                break
+                        if tmp is not None:
+                            tmp[path[-1]] = get_secret(tmp[path[-1]], secretJson)
+                    else:
+                        logger.error(f"invalid path: {path}")
+            
+    return value
+
+def replace_sub_secret(value: Any, field: FieldSchema, secretJson: Dict[str, Any] | None) -> Any:
+    if isinstance(value, dict) and isinstance(field, ObjectFieldSchema):
+        if field.properties is None:
+            return value
+
+        for k, v in value.items():
+            schema = field.properties.get(k)
+            if schema is None:
+                continue
+            if isinstance(schema, SecretFieldSchema):
+                value[k] = get_secret(v, secretJson)
+            elif isinstance(schema, ObjectFieldSchema) or isinstance(schema, ArrayFieldSchema):
+                value[k] = replace_sub_secret(v, schema, secretJson)
+    elif isinstance(value, list) and isinstance(field, ArrayFieldSchema):
+        if field.items is None:
+            return value
+        
+        for i, v in enumerate(value):
+            schema = field.items
+            if isinstance(schema, SecretFieldSchema):
+                value[i] = get_secret(v, secretJson)
+            elif isinstance(schema, ObjectFieldSchema) or isinstance(schema, ArrayFieldSchema):
+                value[i] = replace_sub_secret(v, schema, secretJson)
+            
+    return value
+
+
+def get_secret(path: str, secretJson: dict | None) -> str:
+    if secretJson is None:
+        # throw error
+        logger.error(f"secret file {SECRET_FILE} not found")
+        raise ValueError("secret file not found or invalid json file")
+
+    assert isinstance(secretJson, dict)
+
+    try:
+        [secretType, secretName, secretKey] =  path.split(",")
+    except ValueError:
+        logger.error(f"invalid secret path: {path}")
+        return path
+    
+    s = secretJson.get(secretName)
+
+    if s is None:
+        logger.error(f"secret {secretName} not found in {SECRET_FILE}")
+        return path
+
+    if s.get("secretType") != secretType:
+        logger.warning(f"secret type mismatch: {s.get('secretType')} != {secretType}")
+
+    secrets: list[Any] = s.get("secrets")
+    if secrets:
+        for secret in secrets:
+            if secret.get("secretKey") == secretKey:
+                return secret.get("value")
+    else:
+        logger.error(f"secret {secretName} has no value")
+        return path
+
+    logger.error(f"secret {secretKey} not found in {secretName}")
+    return path

python_executor/service.py

@@ -0,0 +1,238 @@
+from typing import Callable, Any
+from oocana import ServiceExecutePayload, Mainframe, StopAtOption, ServiceContextAbstractClass, ServiceMessage, BlockHandler
+from .block import output_return_object, load_module
+from .context import createContext
+from .utils import run_async_code_and_loop, loop_in_new_thread, run_in_new_thread, oocana_dir
+from .topic import service_config_topic, ServiceTopicParams, ReportStatusPayload, prepare_report_topic, shutdown_action_topic, run_action_topic, service_message_topic, exit_report_topic, status_report_topic,global_shutdown_topic
+from threading import Timer
+import inspect
+import asyncio
+import logging
+import os
+import threading
+
+DEFAULT_BLOCK_ALIVE_TIME = 10
+
+# 两种文件，根据是否有 session id 来区分：
+# 1. session service： ~/.oocana/sessions/{session_id}/python-{service_hash}.log
+# 2. 跨 session service（global service）： ~/.oocana/services/python-{service_hash}.log
+def config_logger(service_hash: str, session_id: str | None):
+    import os.path
+    logger_file = os.path.join(oocana_dir(), "services", "python-" + service_hash + ".log") if session_id is None else os.path.join(oocana_dir(), "sessions", session_id, "python-" + service_hash + ".log") 
+
+    if not os.path.exists(logger_file):
+        os.makedirs(os.path.dirname(logger_file), exist_ok=True)
+
+    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - {%(filename)s:%(lineno)d} - %(message)s', filename=logger_file)
+
+
+class ServiceRuntime(ServiceContextAbstractClass):
+
+    _store = {}
+    _config: ServiceExecutePayload
+    _mainframe: Mainframe
+    _service_hash: str
+    _session_id: str | None = None
+    _timer: Timer | None = None
+    _stop_at: StopAtOption
+    _keep_alive: int | None = None
+    _registered = threading.Event()
+    _waiting_ready_notify = False
+    _session_dir: str
+    _topic_params: ServiceTopicParams
+    _report_timer: Timer | None = None
+
+    _runningBlocks = set()
+    _jobs = set()
+
+    def __init__(self, config: ServiceExecutePayload, mainframe: Mainframe, service_hash: str, session_dir: str, session_id: str | None = None):
+        self._block_handler = dict()
+        self._config = config
+        self._mainframe = mainframe
+        self._service_hash = service_hash
+        self._session_id = session_id
+        self._session_dir = session_dir
+        self._topic_params = {
+            "service_hash": service_hash,
+            "session_id": session_id
+        }
+
+        self._stop_at = config.get("service_executor").get("stop_at") if config.get("service_executor") is not None and config.get("service_executor").get("stop_at") is not None else "session_end"
+        self._keep_alive = config.get("service_executor").get("keep_alive") if config.get("service_executor") is not None else None
+
+        mainframe.subscribe(run_action_topic(self._topic_params), self.run_action_callback)
+        mainframe.subscribe(shutdown_action_topic(self._topic_params), self.shutdown_callback)
+        mainframe.subscribe(global_shutdown_topic, self.shutdown_callback)
+
+        self._setup_timer()
+        self.report_status()
+
+    # post message every 5 seconds
+    def report_status(self):
+        payload: ReportStatusPayload = {
+            "service_hash": self._service_hash,
+            "session_id": self._session_id,
+            "executor": "python"
+        }
+        self._mainframe.publish(status_report_topic(), payload)
+        self._report_timer = Timer(5, self.report_status)
+        self._report_timer.start()
+
+    # 不能直接在 callback 里面调用 mainframe publish 所以 callback 都要单独开
+    def shutdown_callback(self, payload: Any):
+        
+        async def shutdown():
+            self.exit()
+        
+        run_in_new_thread(shutdown)
+
+    def run_action_callback(self, payload: ServiceExecutePayload):
+
+        async def run():
+            await self.run_block(payload)
+        
+        run_in_new_thread(run)
+
+    def _setup_timer(self):
+        if self._stop_at is None:
+            return
+        elif self._stop_at == "session_end":
+            self._mainframe.subscribe("report", lambda payload: self.exit() if payload.get("type") == "SessionFinished" and payload.get("session_id") == self._config.get("session_id") else None)
+        elif self._stop_at == "app_end":
+            # app 暂停可以直接先不管
+            pass
+        elif self._stop_at == "block_end":
+            pass
+
+    def __setitem__(self, key: str, value: Any):
+        if key == "block_handler":
+            self.block_handler = value
+
+    @property
+    def waiting_ready_notify(self) -> bool:
+        return self._waiting_ready_notify
+    
+    @waiting_ready_notify.setter
+    def waiting_ready_notify(self, value: bool):
+        self._waiting_ready_notify = value
+
+    @property
+    def block_handler(self) -> BlockHandler:
+        return self._block_handler
+    
+    @block_handler.setter
+    def block_handler(self, value: BlockHandler):
+        self._block_handler = value
+        if not self.waiting_ready_notify:
+            self._registered.set()
+    
+    def notify_ready(self):
+        self._registered.set()
+
+    def add_message_callback(self, callback: Callable[[ServiceMessage], Any]):
+        def filter(payload):
+            if payload.get("job_id") in self._jobs:
+                callback(payload)
+        self._mainframe.subscribe(service_message_topic, filter)
+
+    async def run(self):
+        service_config = self._config.get("service_executor")
+        m = load_module(service_config.get("entry"), self._config.get("dir"))
+        fn = m.__dict__.get(service_config.get("function"))
+        if not callable(fn):
+            raise Exception(f"function {service_config.get('function')} not found in {service_config.get('entry')}")
+
+        if inspect.iscoroutinefunction(fn):
+            async def async_run():
+                await fn(self)
+            run_in_new_thread(async_run)
+        else:
+            def run():
+                fn(self)
+            import threading
+            threading.Thread(target=run).start()
+    
+        await self.run_block(self._config)
+    
+    def exit(self):
+
+        payload: ReportStatusPayload = {
+            "service_hash": self._service_hash,
+            "session_id": self._session_id,
+            "executor": "python"
+        }
+
+        self._mainframe.publish(exit_report_topic(), payload)
+        self._mainframe.disconnect()
+        # child process need call os._exit not sys.exit
+        os._exit(0)
+
+    async def run_block(self, payload: ServiceExecutePayload):
+        self._registered.wait()
+        block_name = payload["block_name"]
+        job_id = payload["job_id"]
+
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+
+        self._runningBlocks.add(job_id)
+        self._jobs.add(job_id)
+
+        context = createContext(self._mainframe, payload["session_id"], payload["job_id"], self._store, payload["outputs"], self._session_dir)
+
+        if isinstance(self.block_handler, dict):
+            handler = self.block_handler.get(block_name)
+            if handler is None:
+                raise Exception(f"block {block_name} not found")
+            result = handler(context.inputs, context)
+        elif callable(self.block_handler):
+            handler = self.block_handler
+            result = handler(block_name, context.inputs, context)
+        else:
+            raise Exception("blockHandler must be a dict or a callable function")
+        output_return_object(result, context)
+
+        self._runningBlocks.remove(job_id)
+
+        if self._stop_at == "block_end" and len(self._runningBlocks) == 0:
+            self._timer = Timer(self._keep_alive or DEFAULT_BLOCK_ALIVE_TIME, self.exit)
+            self._timer.start()
+
+def setup_service(payload: Any, mainframe: Mainframe, service_id: str, session_dir: str):
+    service = ServiceRuntime(payload, mainframe, service_id, session_dir)
+
+    async def run():
+        await service.run()
+    loop_in_new_thread(run)
+
+
+async def run_service(address: str, service_hash: str, session_id: str | None, session_dir: str):
+    mainframe = Mainframe(address, service_hash)
+    mainframe.connect()
+
+    params: ServiceTopicParams = {
+        "session_id": session_id,
+        "service_hash": service_hash
+    }
+    mainframe.subscribe(service_config_topic(params), lambda payload: setup_service(payload, mainframe, service_hash, session_dir))
+    await asyncio.sleep(1)
+    mainframe.publish(prepare_report_topic(params), {})
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="run service")
+    parser.add_argument("--address", help="mqtt address", required=True)
+    parser.add_argument("--service-hash", help="service hash", required=True)
+    parser.add_argument("--session-id", help="session id")
+    parser.add_argument("--session-dir", required=True)
+    args = parser.parse_args()
+
+    address: str = args.address
+    service_hash: str = args.service_hash
+    session_id: str | None = args.session_id
+    session_dir: str = args.session_dir
+
+    config_logger(service_hash, session_id)
+    run_async_code_and_loop(run_service(address, service_hash, session_id, session_dir))

python_executor/topic.py

@@ -0,0 +1,63 @@
+from typing import Optional, TypedDict
+
+status_topic = "service/report/status"
+exit_topic = "service/report/exit"
+global_shutdown_topic = "service/global/action/shutdown"
+
+class ReportStatusPayload(TypedDict):
+    service_hash: str
+    session_id: Optional[str]
+    executor: str
+
+# prefix
+global_service_topic_prefix = "service/python"
+service_topic_prefix = "/{session_id}/service/python"
+
+# action suffix
+service_config_suffix = "action/config"
+service_run_suffix = "action/run"
+service_ping_suffix = "action/ping"
+service_shutdown_suffix = "action/shutdown"
+
+# report suffix
+service_prepare_suffix = "report/prepare"
+service_pong_suffix = "report/pong"
+
+# topic = {prefix}/{service_hash}/{suffix}
+
+def prefix(session_id: Optional[str] = None) -> str:
+    if session_id:
+        return service_topic_prefix.replace("{session_id}", session_id)
+    else:
+        return global_service_topic_prefix
+
+class ServiceTopicParams(TypedDict):
+    session_id: Optional[str]
+    service_hash: str
+
+def run_action_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_run_suffix])
+
+def ping_action_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_ping_suffix])
+
+def shutdown_action_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_shutdown_suffix])
+
+# TODO: 修改 oomol 端实现，使其为固定的 topic，不需要 session_id
+service_message_topic = "/service/message"
+
+def prepare_report_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_prepare_suffix])
+
+def pong_report_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_pong_suffix])
+
+def exit_report_topic() -> str:
+    return exit_topic
+
+def service_config_topic(params: ServiceTopicParams) -> str:
+    return "/".join([prefix(params.get('session_id')), params['service_hash'], service_config_suffix])
+
+def status_report_topic() -> str:
+    return status_topic

python_executor/utils.py

@@ -0,0 +1,32 @@
+import threading
+import asyncio
+from typing import Callable, Awaitable, Any
+
+def run_async_code(async_func):
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    try:
+        loop.run_until_complete(async_func)
+    finally:
+        loop.close()
+
+def run_async_code_and_loop(async_func):
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    try:
+        loop.run_until_complete(async_func)
+        loop.run_forever()
+    finally:
+        loop.close()
+
+def run_in_new_thread(async_func: Callable[[], Awaitable[Any]]):
+    threading.Thread(target=run_async_code, args=(async_func(),)).start()
+
+def loop_in_new_thread(async_func: Callable[[], Awaitable[Any]]):
+    threading.Thread(target=run_async_code_and_loop, args=(async_func(),)).start()
+
+def oocana_dir() -> str:
+    from os.path import expanduser, join
+    return join(expanduser("~"), ".oocana")