adp-operator 1.0.7__tar.gz

adp_operator-1.0.7/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.4
+Name: adp-operator
+Version: 1.0.7
+Summary: ADP operator management and C2C runtime tools
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: docker
+Requires-Dist: fastapi
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: pyarmor
+Requires-Dist: psutil
+Requires-Dist: python-dotenv
+Requires-Dist: python-multipart
+Requires-Dist: requests
+Requires-Dist: uvicorn
+Requires-Dist: colorlog
+Requires-Dist: websockets
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# ADP Operator
+
+## C2C 连接首包
+
+C2C TCP 连接建立后，双方必须先交换固定长度的二进制首包。首包校验通过后，连接才允许继续处理 JSON 业务指令。
+
+首包固定为 512 bit，结构如下：
+
+| bit 偏移 | bit 长度 | 字段 | 规则 |
+| ---: | ---: | --- | --- |
+| 0 | 24 | 固定标识 | 三个 ASCII 字符 `ADP` |
+| 24 | 16 | 协议版本 | 16-bit 无符号整数，使用大端序，当前版本为 `7` |
+| 40 | 256 | 实例编号 | 当前节点的 `instance_id`，固定32字节 |
+| 296 | 8 | 功能控制区 | bit 0–2为功能开关，bit 3–7为保留位 |
+| 304 | 208 | 全零保留区 | 必须全部为 `0` |
+
+字符串字段统一使用 UTF-8 编码。字段长度不足时在尾部使用 `0x00` 填充，超过字段长度时按字段长度截断。
+
+### 功能控制区
+
+功能控制区固定为8 bit，每个功能占用一个 bit：
+
+| bit | 功能 | `0` | `1` | 当前默认值 |
+| ---: | --- | --- | --- | --- |
+| 0 | UDP 主动发现 | 关闭 | 开启 | `1` |
+| 1 | 额外 channel 通道 | 不支持 | 支持 | `1` |
+| 2 | WebSocket 业务端口 | 关闭 | 开启 | `1` |
+| 3–7 | 保留位 | 必须为零 | 不允许 | `0` |
+
+当前默认控制区的二进制内容为：
+
+```text
+00000111
+```
+
+### 握手流程
+
+1. TCP 客户端连接 C2C 服务端。
+2. 客户端发送自己的 512 bit 二进制首包。
+3. 服务端读取并校验完整首包。
+4. 服务端校验通过后，向客户端返回自己的 512 bit 二进制首包。
+5. 客户端校验服务端首包。
+6. 双方校验均通过后，才开始发送 JSON 业务指令。
+
+服务端会拒绝以下连接：
+
+- 首包长度不足 512 bit；
+- 固定标识不是 `ADP`；
+- 目标节点协议版本低于发起连接节点的协议版本；
+- `instance_id` 为空；
+- 字符串字段的补零格式错误；
+- 功能控制区保留位非零；
+- 208 bit 全零保留区存在非零数据。
+
+该首包适用于 C2C 直连、反向连接、多跳转发、额外 channel 通道、外部 `C2CSession` 和本地服务探测。未实现该二进制首包的旧节点无法与新协议节点建立连接，因此升级时需要同时更新同一网络中的全部 C2C 节点。
+
+协议版本字段固定为16 bit，使用大端序表示无符号整数，取值范围为 `0`–`65535`。客户端收到服务端首包后比较版本：目标节点版本大于或等于本机版本时允许连接，目标节点版本低于本机版本时拒绝连接。例如本机版本为 `4`、目标版本为 `5` 时允许连接；本机版本为 `5`、目标版本为 `4` 时拒绝连接。

adp_operator-1.0.7/README.md

@@ -0,0 +1,57 @@
+# ADP Operator
+
+## C2C 连接首包
+
+C2C TCP 连接建立后，双方必须先交换固定长度的二进制首包。首包校验通过后，连接才允许继续处理 JSON 业务指令。
+
+首包固定为 512 bit，结构如下：
+
+| bit 偏移 | bit 长度 | 字段 | 规则 |
+| ---: | ---: | --- | --- |
+| 0 | 24 | 固定标识 | 三个 ASCII 字符 `ADP` |
+| 24 | 16 | 协议版本 | 16-bit 无符号整数，使用大端序，当前版本为 `7` |
+| 40 | 256 | 实例编号 | 当前节点的 `instance_id`，固定32字节 |
+| 296 | 8 | 功能控制区 | bit 0–2为功能开关，bit 3–7为保留位 |
+| 304 | 208 | 全零保留区 | 必须全部为 `0` |
+
+字符串字段统一使用 UTF-8 编码。字段长度不足时在尾部使用 `0x00` 填充，超过字段长度时按字段长度截断。
+
+### 功能控制区
+
+功能控制区固定为8 bit，每个功能占用一个 bit：
+
+| bit | 功能 | `0` | `1` | 当前默认值 |
+| ---: | --- | --- | --- | --- |
+| 0 | UDP 主动发现 | 关闭 | 开启 | `1` |
+| 1 | 额外 channel 通道 | 不支持 | 支持 | `1` |
+| 2 | WebSocket 业务端口 | 关闭 | 开启 | `1` |
+| 3–7 | 保留位 | 必须为零 | 不允许 | `0` |
+
+当前默认控制区的二进制内容为：
+
+```text
+00000111
+```
+
+### 握手流程
+
+1. TCP 客户端连接 C2C 服务端。
+2. 客户端发送自己的 512 bit 二进制首包。
+3. 服务端读取并校验完整首包。
+4. 服务端校验通过后，向客户端返回自己的 512 bit 二进制首包。
+5. 客户端校验服务端首包。
+6. 双方校验均通过后，才开始发送 JSON 业务指令。
+
+服务端会拒绝以下连接：
+
+- 首包长度不足 512 bit；
+- 固定标识不是 `ADP`；
+- 目标节点协议版本低于发起连接节点的协议版本；
+- `instance_id` 为空；
+- 字符串字段的补零格式错误；
+- 功能控制区保留位非零；
+- 208 bit 全零保留区存在非零数据。
+
+该首包适用于 C2C 直连、反向连接、多跳转发、额外 channel 通道、外部 `C2CSession` 和本地服务探测。未实现该二进制首包的旧节点无法与新协议节点建立连接，因此升级时需要同时更新同一网络中的全部 C2C 节点。
+
+协议版本字段固定为16 bit，使用大端序表示无符号整数，取值范围为 `0`–`65535`。客户端收到服务端首包后比较版本：目标节点版本大于或等于本机版本时允许连接，目标节点版本低于本机版本时拒绝连接。例如本机版本为 `4`、目标版本为 `5` 时允许连接；本机版本为 `5`、目标版本为 `4` 时拒绝连接。

adp_operator-1.0.7/adp_operator/c2c/__init__.py

@@ -0,0 +1,6 @@
+from adp_operator.c2c.client import C2CClient
+from adp_operator.c2c.main import main
+
+
+__all__ = ["C2CClient", "main"]
+

adp_operator-1.0.7/adp_operator/c2c/__main__.py

@@ -0,0 +1,5 @@
+from adp_operator.c2c.main import main
+
+
+main()
+

adp_operator-1.0.7/adp_operator/c2c/client.py

@@ -0,0 +1,128 @@
+import hashlib
+import os
+import sys
+import tempfile
+import threading
+from pathlib import Path
+from typing import Optional
+
+from adp_operator.c2c.operator_executor import OperatorExecutor
+from adp_operator.c2c.runtime import Service
+from adp_operator.c2c.operator_loader import load_operator
+from adp_operator.c2c.operator import C2COperatorBase
+from adp_operator.c2c.operator_source import prepare_operator_source
+
+
+class C2CClient(Service):
+    """一个进程承载一个算子类，按 info/run 指令加载信息或执行算子。"""
+
+    def __init__(
+        self,
+        operator_config: dict,
+        node_id: Optional[str] = None,
+        network: Optional[str] = None,
+        max_concurrent_runs: Optional[int] = None,
+        join: Optional[tuple[str, int]] = None,
+    ):
+        self.operator_config = dict(operator_config)
+        self.operator_base = "PermanentOperator" if join is not None else "ADPOperator"
+        source = Path(operator_config["module_file"]).expanduser().resolve()
+        self._source_config = {**operator_config, "module_file": str(source)}
+        self._workspaces = {}
+        self._workspace_lock = threading.RLock()
+        identity = hashlib.sha256(
+            f"{node_id or operator_config['name']}:{operator_config['module_name']}".encode()
+        ).hexdigest()
+        self._namespace = Path(tempfile.gettempdir()).resolve() / "adp-operator-clients" / identity
+        self._namespace.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self._temporary_directory = tempfile.TemporaryDirectory(prefix="session-", dir=self._namespace)
+        self.working_directory = Path(self._temporary_directory.name)
+        self._previous_directory = Path.cwd()
+        self._import_root = str(self._previous_directory)
+        self._added_import_root = self._import_root not in sys.path
+        try:
+            target = self.working_directory / source.name
+            prepare_operator_source(
+                source, target, operator_config["name"], self.operator_base
+            )
+            self.operator_config["module_file"] = str(target)
+            if self._added_import_root:
+                sys.path.insert(0, self._import_root)
+            os.chdir(self.working_directory)
+            self.operator_class = load_operator(self.operator_config)
+            if not issubclass(self.operator_class, C2COperatorBase):
+                raise TypeError(
+                    f"{self.operator_class.__name__} 必须继承 ADPOperator 或 PermanentOperator"
+                )
+            resolved_node_id = node_id or operator_config["name"]
+            if max_concurrent_runs is None:
+                max_concurrent_runs = max(1, (os.cpu_count() or 1) * 3)
+            executor = OperatorExecutor(
+                self.operator_class, resolved_node_id, max_concurrent_runs=max_concurrent_runs
+            )
+            executor.operator_config = self.operator_config
+            executor.working_directory = self.working_directory
+            executor.get_workspace = self.get_workspace
+            executor.close_workspace = self.close_workspace
+            super().__init__(resolved_node_id, network, executor, join=join)
+        except Exception:
+            self._cleanup_workspace()
+            raise
+
+    def _cleanup_workspace(self):
+        for session_id in list(self._workspaces):
+            self.close_workspace(session_id)
+        if Path.cwd() == self.working_directory:
+            os.chdir(self._previous_directory)
+        if self._added_import_root and self._import_root in sys.path:
+            sys.path.remove(self._import_root)
+        self._temporary_directory.cleanup()
+
+    def get_workspace(self, session_id):
+        """文件接收和算子运行使用同一个会话目录。"""
+        with self._workspace_lock:
+            if session_id not in self._workspaces:
+                if not self._workspaces and self.working_directory.exists():
+                    temporary = self._temporary_directory
+                else:
+                    temporary = tempfile.TemporaryDirectory(prefix="session-", dir=self._namespace)
+                directory = Path(temporary.name)
+                try:
+                    source = Path(self._source_config["module_file"])
+                    target = directory / source.name
+                    prepare_operator_source(
+                        source, target, self._source_config["name"], self.operator_base
+                    )
+                    config = {**self._source_config, "module_file": str(target)}
+                    self._workspaces[session_id] = (temporary, config)
+                except Exception:
+                    temporary.cleanup()
+                    raise
+            return self._workspaces[session_id][1]
+
+    def close_workspace(self, session_id):
+        with self._workspace_lock:
+            workspace = self._workspaces.get(session_id)
+            if workspace is None:
+                return
+            temporary, _ = workspace
+            if Path.cwd() == Path(temporary.name):
+                os.chdir(self._previous_directory)
+            temporary.cleanup()
+            del self._workspaces[session_id]
+
+    def start(self, blocking: bool = True) -> None:
+        try:
+            super().start(blocking=blocking)
+        except Exception:
+            if self._stopped:
+                self._cleanup_workspace()
+            raise
+
+    def stop(self) -> None:
+        super().stop()
+        self._cleanup_workspace()
+
+    def get_extra_info(self):
+        return self.executor.get_extra_info()
+

adp_operator-1.0.7/adp_operator/c2c/commands.py

@@ -0,0 +1,181 @@
+"""与网络入口无关的 C2C 指令实现。"""
+
+import base64
+import logging
+import uuid
+from pathlib import Path
+
+
+def output_event(event, prefix=""):
+    text = event.get("message", event)
+    lines = str(text).splitlines() or [""]
+    if event.get("type") == "log":
+        logger = logging.getLogger(event.get("logger") or __name__)
+        level = getattr(logging, str(event.get("level", "INFO")).upper(), logging.INFO)
+        if not isinstance(level, int):
+            level = logging.INFO
+        logger.disabled = False
+        if logger.getEffectiveLevel() > level:
+            logger.setLevel(level)
+        if not logger.hasHandlers():
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
+            logger.addHandler(handler)
+            logger.propagate = False
+        logger.log(level, "\n".join(f"{prefix}{line}" for line in lines))
+        return
+    print("\n".join(f"{prefix}{line}" for line in lines), flush=True)
+
+
+class C2CCommands:
+    """统一实现 C2C 的运行、调用和文件指令。"""
+
+    def __init__(self, transport, session_id: str | None = None) -> None:
+        self.transport = transport
+        self.session_id = session_id or uuid.uuid4().hex
+        self._operators: set[str] = set()
+        self._closed = False
+
+    def can_reach(self, operator_name: str) -> bool:
+        return self.transport.can_reach(operator_name, self.session_id)
+
+    def call(self, operator_name: str, method: str, *args, **kwargs):
+        if not self.can_reach(operator_name):
+            return None
+        result = None
+        result_received = False
+        for event in self._request(operator_name, {
+            "type": "call",
+            "operator_name": operator_name,
+            "method": method,
+            "args": list(args),
+            "kwargs": kwargs,
+        }, {"call_end", "call_error"}):
+            if event.get("type") == "call_error":
+                raise RuntimeError(event.get("error", "C2C 方法调用失败"))
+            if event.get("type") == "call_result":
+                result = event.get("result")
+                result_received = True
+            if event.get("type") == "call_end":
+                if not result_received:
+                    raise RuntimeError(f"方法 {operator_name}.{method} 未返回结果")
+                return result
+        raise RuntimeError(f"方法 {operator_name}.{method} 的事件流异常结束")
+
+    def run(self, message: dict, on_event=None):
+        operator_name = message["operator_name"]
+        result = None
+        for event in self._request(
+            operator_name, message, {"run_end", "run_error"}
+        ):
+            event_type = event.get("type")
+            if event_type == "log":
+                continue
+            if event_type == "run_end" and result is None:
+                event = {
+                    "type": "run_error",
+                    "status": "error",
+                    "operator_name": operator_name,
+                    "pipeline_id": message.get("pipeline_id"),
+                    "error": f"算子 {operator_name} 运行失败，未返回结果",
+                }
+                event_type = "run_error"
+            if event_type in ("run_result", "run_error"):
+                result = event
+            if on_event is None:
+                output_event(event)
+            else:
+                on_event(event)
+            if event_type == "run_error":
+                return result
+            if event_type == "run_end":
+                return result
+        raise RuntimeError(f"算子 {operator_name} 的事件流异常结束")
+
+    def transfer_file(self, operator_name: str, path, filename: str) -> str:
+        offset = 0
+        response = None
+        with Path(path).open("rb") as source:
+            for chunk in iter(lambda: source.read(512 * 1024), b""):
+                response = self._single(operator_name, {
+                    "type": "receive_file",
+                    "operator_name": operator_name,
+                    "filename": filename,
+                    "offset": offset,
+                    "content": base64.b64encode(chunk).decode("ascii"),
+                }, "receive_file_ack")
+                offset += len(chunk)
+                if response.get("size") != offset:
+                    raise RuntimeError("文件接收确认不一致")
+        if response is None:
+            response = self._single(operator_name, {
+                "type": "receive_file",
+                "operator_name": operator_name,
+                "filename": filename,
+                "offset": 0,
+                "content": "",
+            }, "receive_file_ack")
+            if response.get("size") != 0:
+                raise RuntimeError("空文件接收确认不一致")
+        return response["path"]
+
+    def sync_files(self, operator_name: str, targets: list[str]) -> dict:
+        targets = list(dict.fromkeys(targets))
+        if not targets:
+            return {}
+        self._operators.update(targets)
+        response = self._single(operator_name, {
+            "type": "sync_files",
+            "operator_name": operator_name,
+            "targets": targets,
+        }, "sync_files_ack")
+        return response["synchronized"]
+
+    def close(self) -> None:
+        if self._closed:
+            return
+        error = None
+        try:
+            for operator_name in self._operators:
+                try:
+                    self._single(operator_name, {
+                        "type": "close_session",
+                        "operator_name": operator_name,
+                    }, "close_session_ack")
+                except Exception as exc:
+                    error = error or exc
+        finally:
+            self._closed = True
+            self.transport.close()
+        if error is not None:
+            raise error
+
+    def _single(self, operator_name: str, message: dict, response_type: str) -> dict:
+        response = None
+        for event in self._request(operator_name, message, {response_type}):
+            response = event
+        if response is None or response.get("type") != response_type:
+            raise RuntimeError(f"{message['type']} 响应无效")
+        if response.get("status") == "error":
+            raise RuntimeError(response.get("error", "C2C 请求失败"))
+        return response
+
+    def _request(
+        self,
+        operator_name: str,
+        message: dict,
+        terminal_types: set[str],
+    ):
+        self._operators.add(operator_name)
+        yield from self.transport.request(
+            operator_name,
+            message,
+            self.session_id,
+            terminal_types,
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

adp_operator-1.0.7/adp_operator/c2c/config.py

@@ -0,0 +1,11 @@
+BROADCAST_BASE_PORT = 50000
+BROADCAST_PORT_RANGE = 1000
+SERVICE_BASE_PORT = 60000
+SERVICE_PORT_RANGE = 1000
+WEBSOCKET_BASE_PORT = 61000
+WEBSOCKET_PORT_RANGE = 1000
+BROADCAST_INTERVAL = 3
+PEER_SYNC_INTERVAL = 3
+PEER_TTL = 300
+MAX_REQUEST_SIZE = 1024 * 1024
+