oocana-python-executor 0.15.4__tar.gz → 0.16.1__tar.gz

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/PKG-INFO

@@ -1,9 +1,11 @@
 Metadata-Version: 2.1
 Name: oocana-python-executor
-Version: 0.15.4
+Version: 0.16.1
 Summary: a client subscribe mqtt topic to execute oocana's block
 Author-Email: l1shen <lishen1635@gmail.com>, yleaf <11785335+leavesster@users.noreply.github.com>
 License: MIT
 Requires-Python: >=3.9
 Requires-Dist: oocana
+Provides-Extra: debug
+Requires-Dist: debugpy; extra == "debug"
 

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oocana-python-executor"
-version = "0.15.4"
+version = "0.16.1"
 authors = [
     { name = "l1shen", email = "lishen1635@gmail.com" },
     { name = "yleaf", email = "11785335+leavesster@users.noreply.github.com" },
@@ -14,6 +14,11 @@ dependencies = [
 [project.license]
 text = "MIT"
 
+[project.optional-dependencies]
+debug = [
+    "debugpy",
+]
+
 [project.scripts]
 python-executor = "python_executor.executor:main"
 

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/block.py

@@ -4,7 +4,7 @@ from typing import Optional, TypedDict
 import inspect
 import traceback
 import logging
-from .data import store, vars, EXECUTOR_NAME
+from .data import store, block_var, EXECUTOR_NAME
 from .context import createContext
 from .hook import ExitFunctionException
 import os
@@ -70,24 +70,24 @@ def load_module(file_path: str, source_dir=None):
 
 def output_return_object(obj, context: Context):
     if obj is None:
-        context.done()
+        context.done() if not context.is_done else None
     elif obj is context.keepAlive:
         pass
     elif isinstance(obj, dict):
         for k, v in obj.items():
             context.output(k, v)
-        context.done()
+        context.done() if not context.is_done else None
     else:
         context.done(f"return object needs to be a dictionary, but get type: {type(obj)}")
 
 logger = logging.getLogger(EXECUTOR_NAME)
 
-async def run_block(message, mainframe: Mainframe, session_dir: str):
+async def run_block(message, mainframe: Mainframe, session_dir: str, tmp_dir: str):
 
     logger.info(f"block {message.get('job_id')} start")
     try:
         payload = ExecutePayload(**message)
-        context = createContext(mainframe, payload.session_id, payload.job_id, store, payload.outputs, session_dir)
+        context = createContext(mainframe, payload.session_id, payload.job_id, store, payload.outputs, session_dir, tmp_dir = tmp_dir)
     except Exception:
         traceback_str = traceback.format_exc()
         # rust 那边会保证传过来的 message 一定是符合格式的，所以这里不应该出现异常。这里主要是防止 rust 修改错误。
@@ -103,7 +103,7 @@ async def run_block(message, mainframe: Mainframe, session_dir: str):
         })
         return
 
-    vars.set(context)
+    block_var.set(context)
 
     load_dir = payload.dir
 

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/context.py

@@ -9,7 +9,7 @@ from .data import EXECUTOR_NAME
 logger = logging.getLogger(EXECUTOR_NAME)
 
 def createContext(
-    mainframe: Mainframe, session_id: str, job_id: str, store, output, session_dir: str
+    mainframe: Mainframe, session_id: str, job_id: str, store, output, session_dir: str, tmp_dir: str
 ) -> Context:
 
     node_props = mainframe.notify_block_ready(session_id, job_id)
@@ -59,7 +59,7 @@ def createContext(
     
     blockInfo = BlockInfo(**node_props)
 
-    ctx = Context(inputs, blockInfo, mainframe, store, output, session_dir)
+    ctx = Context(inputs, blockInfo, mainframe, store, output, session_dir, tmp_dir)
     # 跟 executor 日志分开，避免有的库在 logger 里面使用 print，导致 hook 出现递归调用。
     block_logger = logging.getLogger(f"block {job_id}")
     ctx_handler = ContextHandler(ctx)

oocana_python_executor-0.16.1/python_executor/data.py

@@ -0,0 +1,5 @@
+from contextvars import ContextVar
+from oocana import Context, EXECUTOR_NAME  # noqa: F401
+
+block_var: ContextVar[Context] = ContextVar('block-context')
+store = {}

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/executor.py

@@ -49,7 +49,7 @@ def config_logger(session_id: str, identifier: str | None, output: Literal["cons
     logger.propagate = False
 
 
-async def run_executor(address: str, session_id: str, package: str | None, session_dir: str, identifier: str | None = None):
+async def run_executor(address: str, session_id: str, tmp_dir: str, package: str | None, session_dir: str, identifier: str | None = None):
 
     if identifier is not None:
         mainframe = Mainframe(address, f"python-executor-id-{identifier}", logger)
@@ -211,12 +211,12 @@ async def run_executor(address: str, session_id: str, package: str | None, sessi
             else:
                 if not_current_session(message):
                     continue
-                run_block_in_new_thread(message, mainframe, session_dir=session_dir)
+                run_block_in_new_thread(message, mainframe, session_dir=session_dir, tmp_dir=tmp_dir)
 
-def run_block_in_new_thread(message, mainframe: Mainframe, session_dir: str):
+def run_block_in_new_thread(message, mainframe: Mainframe, session_dir: str, tmp_dir: str):
 
     async def run():
-        await run_block(message, mainframe, session_dir=session_dir)
+        await run_block(message, mainframe, session_dir=session_dir, tmp_dir=tmp_dir)
     run_in_new_thread(run)
 
 def main():
@@ -224,30 +224,51 @@ def main():
     import argparse
     parser = argparse.ArgumentParser(description="run executor with address, session-id, tmp-dir")
     parser.add_argument("--session-id", help="executor subscribe session id", required=True)
-    parser.add_argument("--address", help="mqtt address", default="mqtt://127.0.0.1:47688")
     parser.add_argument("--session-dir", help="a tmp dir for whole session", required=True)
+    parser.add_argument("--tmp-dir", help="a tmp dir for whole session. It will be cleaned after session success, this behavior is guaranteed by oocana.", required=True)
+    parser.add_argument("--address", help="mqtt address", default="mqtt://127.0.0.1:47688")
     parser.add_argument("--output", help="output log to console or file", default="file", choices=["console", "file"])
     parser.add_argument("--package", help="package path, if set, executor will only run same package block", default=None)
     parser.add_argument("--identifier", help="identifier for executor, oocana will think same identifier as one executor", default=None)
+    parser.add_argument("--debug-port", help="debug port for python", default=None)
+    parser.add_argument("--wait-for-client", help="wait for client to connect", default=False, action="store_true")
 
     try:
-        args = parser.parse_args()
+        namespace, unknown_args = parser.parse_known_args()
     except Exception as e:
-        print(f"parse args error: {e}")
+        hook.original_print(f"parse args error: {e}")
         # because we hook sys.exit in hook.py and raise a exception, the exit will be reset to 1.
         # parser origin exit code is 2. so we use 2 here.
-        sys.exit(2)
+        hook.original_exit(2)
 
-    address: str = args.address
-    session_id: str = str(args.session_id)
-    output: Literal["console", "file"] = args.output
-    package: str | None = args.package
-    session_dir: str = args.session_dir
-    identifier: str | None = args.identifier
+    address: str = namespace.address
+    session_id: str = str(namespace.session_id)
+    output: Literal["console", "file"] = namespace.output
+    package: str | None = namespace.package
+    session_dir: str = namespace.session_dir
+    tmp_dir: str = namespace.tmp_dir
+    identifier: str | None = namespace.identifier
 
     config_logger(session_id, identifier, output)
 
-    run_async_code(run_executor(address=address, session_id=session_id, package=package, session_dir=session_dir, identifier=identifier))
+    if len(unknown_args) > 0:
+        logger.warning(f"receive unknown args: {unknown_args}")
+
+    if namespace.debug_port is not None and namespace.debug_port.isdigit():
+        try:
+            import debugpy
+            debugpy.listen(int(namespace.debug_port))
+            logger.info(f"debugpy listen on port {namespace.debug_port}")
+            if namespace.wait_for_client:
+                logger.info("wait for client to connect")
+                debugpy.wait_for_client()
+                logger.info("client connected")
+        except ImportError:
+            logger.warning("Warning: debugpy not installed, debugging functionality will not be available")
+        except Exception as e:
+            logger.warning(f"Warning: debugpy listen failed: {e}")
+
+    run_async_code(run_executor(address=address, tmp_dir=tmp_dir, session_id=session_id, package=package, session_dir=session_dir, identifier=identifier))
 
 if __name__ == '__main__':
     main()

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/hook.py

@@ -3,7 +3,7 @@ from builtins import exit as global_exit
 from typing import TypeAlias, Any
 import sys
 import builtins
-from .data import vars, EXECUTOR_NAME
+from .data import block_var, EXECUTOR_NAME
 import logging
 
 logger = logging.getLogger(EXECUTOR_NAME)
@@ -18,27 +18,25 @@ original_print = print
 _ExitCode: TypeAlias = str | int | None
 
 def sys_exit(status: _ExitCode = None) -> None:
-    raise ExitFunctionException(status)
+    if block_var.get(None) is not None:
+        raise ExitFunctionException(status)
+    else:
+        original_exit(status)
 
 def sys_global_exit(status: _ExitCode = None) -> None:
-    raise ExitFunctionException(status)
+    if block_var.get(None) is not None:
+        raise ExitFunctionException(status)
+    else:
+        original_global_exit(status)
 
 def global_print(*values: object, sep: str | None = " ", end: str | None = "\n", file: Any | None = None, flush: bool = False) -> None:
-    
-    context = None  # 初始化 context 变量
-    try:
-        context = vars.get()
-    except LookupError:
-        # 这个 logger 不会上报到 root handle 中，所以即使 root logger 的 Handler 里面有 print 函数，也不会导致递归调用
-        logger.warning("print called outside of block")
-    except Exception as e:
-        logger.error(f"print error: {e}")
-
-    if context is not None:
+
+    block_ctx = block_var.get(None)
+    if block_ctx is not None:
         try:
             msg_sep = sep or " "
             msg = msg_sep.join(map(str, values))
-            context.report_log(msg)
+            block_ctx.report_log(msg)
         except Exception as e:
             logger.error(f"transform print message to context log error: {e}")
 

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/matplot/matplotlib_oomol/oomol.py

@@ -2,7 +2,7 @@
 
 from matplotlib.backend_bases import Gcf # type: ignore
 from matplotlib.backends.backend_agg import FigureCanvasAgg # type: ignore
-from python_executor.data import vars
+from python_executor.data import block_var
 
 FigureCanvas = FigureCanvasAgg
 
@@ -10,8 +10,8 @@ def show(*args, **kwargs):
     import sys
     from io import BytesIO
     from base64 import b64encode
-    if vars is not None:
-        context = vars.get()
+    context = block_var.get(None)
+    if context is not None:
         images = []
         for figmanager in Gcf.get_all_fig_managers():
                 buffer = BytesIO()

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/matplot/oomol_matplot_helper.py

@@ -1,4 +1,4 @@
-from python_executor.data import vars
+from python_executor.data import block_var
 
 def add_matplot_module():
     import sys
@@ -13,6 +13,7 @@ def import_helper(logger):
         matplotlib.use('module://matplotlib_oomol') # matplotlib_oomol.py 文件所在目录加入 PYTHONPATH
     except:
         logger.error("import matplotlib failed")
+        return
 
     # matplotlib 主题替换
     try:
@@ -34,8 +35,8 @@ def import_helper(logger):
 
         class OomolRenderer(ExternalRenderer):
             def render(self, fig_dict):
-                if vars:
-                    context = vars.get()
+                context = block_var.get(None)
+                if context is not None:
 
                     import re
                     from plotly.io import to_html # type: ignore

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/python_executor/service.py

@@ -179,7 +179,7 @@ class ServiceRuntime(ServiceContextAbstractClass):
         self._runningBlocks.add(job_id)
         self._jobs.add(job_id)
 
-        context = createContext(self._mainframe, payload["session_id"], payload["job_id"], self._store, payload["outputs"], self._session_dir)
+        context = createContext(self._mainframe, payload["session_id"], payload["job_id"], self._store, payload["outputs"], self._session_dir, tmp_dir=self._session_dir) # TODO: tmp_dir need consider global service.
 
         if isinstance(self.block_handler, dict):
             handler = self.block_handler.get(block_name)

{oocana_python_executor-0.15.4 → oocana_python_executor-0.16.1}/tests/test_cli.py

@@ -20,7 +20,24 @@ class TestExecutorCLI(unittest.TestCase):
         self.assertIsNone(code, "HTTP server failed to start or exit.")
 
     def test_cli(self):
-        cli_command = [sys.executable, "-u", "-m", "python_executor.executor", "--session-id", "test-session", "--session-dir", "/tmp"]
+        cli_command = [sys.executable, "-u", "-m", "python_executor.executor", "--session-id", "test-session", "--session-dir", "/tmp", "--tmp-dir", "/tmp"]
+
+        print("Starting CLI tool... in", executor_parent_dir)
+
+        process = subprocess.Popen(cli_command, cwd=executor_parent_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        time.sleep(3)
+
+        code = process.poll()
+
+        self.assertIsNone(code, f"CLI tool failed to start or exit with code {code}")
+
+        process.send_signal(signal.SIGINT)
+
+        process.wait()
+
+    def test_cli_addition_args(self):
+        cli_command = [sys.executable, "-u", "-m", "python_executor.executor", "--session-id", "test-session", "--session-dir", "/tmp", "--tmp-dir", "/tmp", "--echo", "Hello World!"]
 
         print("Starting CLI tool... in", executor_parent_dir)
 

oocana_python_executor-0.15.4/python_executor/data.py

@@ -1,5 +0,0 @@
-from contextvars import ContextVar
-from oocana import Context, EXECUTOR_NAME
-
-vars: ContextVar[Context] = ContextVar('context')
-store = {}