octoparse-dataapp 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- octoparse_dataapp/__init__.py +24 -0
- octoparse_dataapp/_abort.py +119 -0
- octoparse_dataapp/_app.py +382 -0
- octoparse_dataapp/_backend.py +53 -0
- octoparse_dataapp/_brand.py +4 -0
- octoparse_dataapp/_errors.py +29 -0
- octoparse_dataapp/_local.py +208 -0
- octoparse_dataapp/_log.py +68 -0
- octoparse_dataapp/_outcome.py +68 -0
- octoparse_dataapp/_platform.py +271 -0
- octoparse_dataapp/_progress.py +66 -0
- octoparse_dataapp/_state.py +16 -0
- octoparse_dataapp/_writer.py +188 -0
- octoparse_dataapp/py.typed +0 -0
- octoparse_dataapp-0.0.2.dist-info/METADATA +140 -0
- octoparse_dataapp-0.0.2.dist-info/RECORD +18 -0
- octoparse_dataapp-0.0.2.dist-info/WHEEL +4 -0
- octoparse_dataapp-0.0.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""octoparse-dataapp:开发 Octoparse DataHub Data App 的 SDK。
|
|
2
|
+
|
|
3
|
+
进程归作者、SDK 是库:作者的 app.py 是入口,在 `async with DataApp:` 内
|
|
4
|
+
读取输入、写回记录、响应取消。同一份代码本地与平台都用 `uv run python app.py`
|
|
5
|
+
运行;存在 DATAHUB_SERVER_URL 与 DATAHUB_WORKER_TOKEN 即平台模式(按平台 worker 通道
|
|
6
|
+
协议认领 run),否则本地模式。
|
|
7
|
+
|
|
8
|
+
调用其他 Data App(发现、运行、取结果)属于用户侧能力,见 octoparse-client。
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
12
|
+
|
|
13
|
+
from ._brand import BRAND
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
__version__ = version(f"{BRAND}-dataapp")
|
|
17
|
+
except PackageNotFoundError: # 源码树直接导入(未安装)
|
|
18
|
+
__version__ = "0.0.0"
|
|
19
|
+
|
|
20
|
+
from ._app import AbortReason, DataApp, Event # noqa: E402
|
|
21
|
+
from ._errors import DataAppError, RunTerminatedError # noqa: E402
|
|
22
|
+
|
|
23
|
+
__all__ = ["DataApp", "Event", "AbortReason", "DataAppError", "RunTerminatedError",
|
|
24
|
+
"__version__"]
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""ABORTING:取消 / 超时 / 条数熔断 / SIGTERM·SIGINT 统一走这里,只触发一次。
|
|
2
|
+
|
|
3
|
+
有作者 handler 就交给作者收尾;没有就由 SDK 调 exit()。
|
|
4
|
+
第二次信号不再等作者,直接取消主任务。
|
|
5
|
+
超时(TIMEOUT)另有宽限:作者 handler 跑完后 TIMEOUT_GRACE_SECONDS 内 run 还没结束,
|
|
6
|
+
SDK 直接结束主任务——否则不配合收尾的作者代码会一直跑到执行环境生命期耗尽才被平台收尸。
|
|
7
|
+
取消 / 熔断仍是协作式的,与引擎自己的看门狗同口径(只有超时强停)。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import inspect
|
|
14
|
+
import signal
|
|
15
|
+
from collections.abc import Awaitable, Callable
|
|
16
|
+
from enum import Enum
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from ._log import sdk_logger as logger
|
|
20
|
+
from ._outcome import AbortReason
|
|
21
|
+
from ._state import RunState
|
|
22
|
+
|
|
23
|
+
# 超时后给作者收尾的宽限(秒):到点由 SDK 结束主任务;SDK 自己的攒批写出与终态上报在其后照常进行
|
|
24
|
+
TIMEOUT_GRACE_SECONDS = 15.0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Event(str, Enum):
|
|
28
|
+
ABORTING = "aborting"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
Handler = Callable[[dict[str, Any]], None | Awaitable[None]]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AbortController:
|
|
35
|
+
def __init__(self, state: RunState, default_action: Callable[[], Awaitable[None]],
|
|
36
|
+
force_stop: Callable[[], Awaitable[None]] | None = None) -> None:
|
|
37
|
+
self._state = state
|
|
38
|
+
self._default_action = default_action # 没有 ABORTING handler 时的收尾(= exit())
|
|
39
|
+
self._force_stop = force_stop or default_action # 超时宽限到点的强停(= 结束主任务)
|
|
40
|
+
self._handlers: dict[Event, list[Handler]] = {}
|
|
41
|
+
self._task: asyncio.Task[None] | None = None
|
|
42
|
+
self._main_task: asyncio.Task[Any] | None = None
|
|
43
|
+
self._signals: list[int] = []
|
|
44
|
+
self.reason: AbortReason | None = None
|
|
45
|
+
self.cancel_requested = False
|
|
46
|
+
|
|
47
|
+
def on(self, event: Event, handler: Handler) -> None:
|
|
48
|
+
self._handlers.setdefault(Event(event), []).append(handler)
|
|
49
|
+
|
|
50
|
+
def trigger(self, reason: AbortReason) -> None:
|
|
51
|
+
if self.reason is not None or self._state.finished:
|
|
52
|
+
return
|
|
53
|
+
self.reason = reason
|
|
54
|
+
logger.info("aborting (%s)", reason.value)
|
|
55
|
+
self._task = asyncio.create_task(self._run_handlers(reason), name="dataapp-aborting")
|
|
56
|
+
|
|
57
|
+
def request_cancel(self) -> None:
|
|
58
|
+
"""平台下发了取消信号(进度响应捎带或轮询得到)。"""
|
|
59
|
+
self.cancel_requested = True
|
|
60
|
+
self.trigger(AbortReason.CANCEL)
|
|
61
|
+
|
|
62
|
+
def ensure_reason(self, reason: AbortReason) -> None:
|
|
63
|
+
"""只记原因不跑 handler:主任务已在收尾(外部取消、Ctrl-C)时用。"""
|
|
64
|
+
if self.reason is None:
|
|
65
|
+
self.reason = reason
|
|
66
|
+
|
|
67
|
+
async def _run_handlers(self, reason: AbortReason) -> None:
|
|
68
|
+
handlers = list(self._handlers.get(Event.ABORTING, ()))
|
|
69
|
+
if not handlers:
|
|
70
|
+
await self._default_action()
|
|
71
|
+
return
|
|
72
|
+
for handler in handlers:
|
|
73
|
+
try:
|
|
74
|
+
result = handler({"reason": reason.value})
|
|
75
|
+
if inspect.isawaitable(result):
|
|
76
|
+
await result
|
|
77
|
+
except asyncio.CancelledError:
|
|
78
|
+
raise
|
|
79
|
+
except Exception: # noqa: BLE001 — 作者代码
|
|
80
|
+
logger.exception("ABORTING handler failed")
|
|
81
|
+
if reason is AbortReason.TIMEOUT and not self._state.finished:
|
|
82
|
+
# 作者 handler 跑完却没结束 run:宽限后由 SDK 结束主任务,终态照常是 run-timeout
|
|
83
|
+
await asyncio.sleep(TIMEOUT_GRACE_SECONDS)
|
|
84
|
+
if not self._state.finished:
|
|
85
|
+
logger.warning("run did not stop within %.0fs after the timeout; stopping it",
|
|
86
|
+
TIMEOUT_GRACE_SECONDS)
|
|
87
|
+
await self._force_stop()
|
|
88
|
+
|
|
89
|
+
# ── 信号 ─────────────────────────────────────────────────
|
|
90
|
+
def install_signals(self, main_task: asyncio.Task[Any] | None) -> None:
|
|
91
|
+
self._main_task = main_task
|
|
92
|
+
loop = asyncio.get_running_loop()
|
|
93
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
94
|
+
try:
|
|
95
|
+
loop.add_signal_handler(sig, self._on_signal)
|
|
96
|
+
except (NotImplementedError, RuntimeError, ValueError):
|
|
97
|
+
continue # Windows / 非主线程:没有信号处理也能跑,只是收不到 ABORTING
|
|
98
|
+
self._signals.append(sig)
|
|
99
|
+
|
|
100
|
+
def _on_signal(self) -> None:
|
|
101
|
+
if self.reason is not None and self._main_task is not None:
|
|
102
|
+
# 第二次信号:不再等作者收尾,直接取消主任务
|
|
103
|
+
self._main_task.cancel()
|
|
104
|
+
return
|
|
105
|
+
self.trigger(AbortReason.SIGNAL)
|
|
106
|
+
|
|
107
|
+
async def close(self) -> None:
|
|
108
|
+
loop = asyncio.get_running_loop()
|
|
109
|
+
for sig in self._signals:
|
|
110
|
+
loop.remove_signal_handler(sig)
|
|
111
|
+
self._signals.clear()
|
|
112
|
+
task = self._task
|
|
113
|
+
# 在 ABORTING 任务里调 exit() 走到这里时不能取消自己
|
|
114
|
+
if task is not None and task is not asyncio.current_task() and not task.done():
|
|
115
|
+
task.cancel()
|
|
116
|
+
try:
|
|
117
|
+
await task
|
|
118
|
+
except (asyncio.CancelledError, Exception):
|
|
119
|
+
pass
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""DataApp 门面:进程归作者,SDK 是库。
|
|
2
|
+
|
|
3
|
+
async with DataApp: # 进入:拿到 run 与输入
|
|
4
|
+
inp = await DataApp.get_input()
|
|
5
|
+
await DataApp.push_data({...})
|
|
6
|
+
# 退出:上报终态(正常 = succeeded,异常 = failed)
|
|
7
|
+
|
|
8
|
+
门面只做编排:写回管道在 _writer,ABORTING 与信号在 _abort,进度节流在 _progress,
|
|
9
|
+
终态归约在 _outcome,平台/本地差异在后端。生命周期收尾(__aexit__ / exit() / fail()
|
|
10
|
+
与主任务取消)刻意留在本文件:它们的正确性依赖 await 前后的置位顺序。
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import logging
|
|
17
|
+
import time
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from . import _log, _outcome
|
|
22
|
+
from ._abort import AbortController, Event, Handler
|
|
23
|
+
from ._backend import Backend, RunInfo
|
|
24
|
+
from ._errors import DataAppError, PushRejectedError, RunTerminatedError
|
|
25
|
+
from ._local import LocalBackend
|
|
26
|
+
from ._outcome import AbortReason
|
|
27
|
+
from ._platform import (
|
|
28
|
+
ENV_PREFIX,
|
|
29
|
+
EXIT_OK,
|
|
30
|
+
EXIT_PLATFORM_UNAVAILABLE,
|
|
31
|
+
EXIT_USAGE,
|
|
32
|
+
NothingToRun,
|
|
33
|
+
PlatformBackend,
|
|
34
|
+
PlatformUnavailable,
|
|
35
|
+
platform_env_partial,
|
|
36
|
+
platform_env_present,
|
|
37
|
+
)
|
|
38
|
+
from ._progress import ProgressReporter
|
|
39
|
+
from ._state import RunState
|
|
40
|
+
from ._writer import RecordWriter, encoded_size
|
|
41
|
+
|
|
42
|
+
__all__ = ["AbortReason", "DataApp", "Event"]
|
|
43
|
+
|
|
44
|
+
logger = _log.sdk_logger
|
|
45
|
+
|
|
46
|
+
CANCEL_POLL_INTERVAL = 2.0 # 取消信号轮询(秒);平台侧兼作心跳
|
|
47
|
+
TICK = 0.25
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class _DataApp:
|
|
51
|
+
def __init__(self) -> None:
|
|
52
|
+
self._reset()
|
|
53
|
+
|
|
54
|
+
def _reset(self) -> None:
|
|
55
|
+
self._state = RunState()
|
|
56
|
+
# 先于 async with 创建:作者可以在进入前就 on(ABORTING, ...)
|
|
57
|
+
self._abort = AbortController(self._state, self.exit, self._force_stop)
|
|
58
|
+
self._backend: Backend | None = None
|
|
59
|
+
self._info: RunInfo | None = None
|
|
60
|
+
self._writer: RecordWriter | None = None
|
|
61
|
+
self._progress: ProgressReporter | None = None
|
|
62
|
+
self._main_task: asyncio.Task[Any] | None = None
|
|
63
|
+
self._ticker: asyncio.Task[None] | None = None
|
|
64
|
+
self._stopping = False # exit()/fail() 主动结束了主任务
|
|
65
|
+
self._exiting = False # 主任务已进入 __aexit__:此后任何路径都不得再取消它
|
|
66
|
+
# 收尾单飞:谁先调 _finalize 谁创建这个任务,后来者只等它。独立成任务是为了
|
|
67
|
+
# 它不随发起方(ABORTING 任务 / 主任务 / 节拍)被取消而半途而废
|
|
68
|
+
self._finalizing: asyncio.Task[None] | None = None
|
|
69
|
+
self._outcome: str | None = None
|
|
70
|
+
self._last_cancel_poll = 0.0
|
|
71
|
+
|
|
72
|
+
# ── 生命周期 ─────────────────────────────────────────────
|
|
73
|
+
async def __aenter__(self) -> _DataApp:
|
|
74
|
+
if self._backend is not None:
|
|
75
|
+
raise RuntimeError("DataApp is already running")
|
|
76
|
+
_log.get_logger()
|
|
77
|
+
self._main_task = asyncio.current_task()
|
|
78
|
+
if platform_env_partial():
|
|
79
|
+
# 平台总是成对注入;缺一个只能是镜像 wrapper 吞掉了环境——不能悄悄落到本地模式
|
|
80
|
+
logger.error("incomplete platform environment: %sSERVER_URL and %sWORKER_TOKEN "
|
|
81
|
+
"must both be set", ENV_PREFIX, ENV_PREFIX)
|
|
82
|
+
raise SystemExit(EXIT_USAGE)
|
|
83
|
+
backend = self._backend = PlatformBackend() if platform_env_present() \
|
|
84
|
+
else LocalBackend(Path.cwd())
|
|
85
|
+
try:
|
|
86
|
+
self._info = await backend.start()
|
|
87
|
+
except BaseException as exc:
|
|
88
|
+
await backend.close()
|
|
89
|
+
self._reset()
|
|
90
|
+
# 认领阶段的结局按运行时协议的退出码交给平台归因;with 块体不执行
|
|
91
|
+
if isinstance(exc, NothingToRun):
|
|
92
|
+
logger.info("%s; exiting", exc)
|
|
93
|
+
raise SystemExit(EXIT_OK) from None
|
|
94
|
+
if isinstance(exc, PlatformUnavailable):
|
|
95
|
+
logger.error("platform unavailable: %s", exc)
|
|
96
|
+
raise SystemExit(EXIT_PLATFORM_UNAVAILABLE) from None
|
|
97
|
+
raise
|
|
98
|
+
self._writer = RecordWriter(backend, self._state, self._info.max_records)
|
|
99
|
+
self._progress = ProgressReporter(backend, self._state, self._abort.request_cancel)
|
|
100
|
+
_log.set_secrets(list(self._info.env.values()))
|
|
101
|
+
self._abort.install_signals(self._main_task)
|
|
102
|
+
self._ticker = asyncio.create_task(self._tick_loop(), name="dataapp-ticker")
|
|
103
|
+
return self
|
|
104
|
+
|
|
105
|
+
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
|
|
106
|
+
# 先于一切 await 置位:ABORTING 任务(熔断/取消/超时)可能恰在主任务收尾时
|
|
107
|
+
# 调 exit(),那时再取消主任务,CancelledError 会从 __aexit__ 里冒出去
|
|
108
|
+
self._exiting = True
|
|
109
|
+
suppress = False
|
|
110
|
+
failed_locally = False
|
|
111
|
+
try:
|
|
112
|
+
if self._stopping and exc_type is not None \
|
|
113
|
+
and issubclass(exc_type, asyncio.CancelledError):
|
|
114
|
+
# exit()/fail() 取消了主任务:这是预期的收尾路径
|
|
115
|
+
suppress = True
|
|
116
|
+
task = self._main_task
|
|
117
|
+
if task is not None and hasattr(task, "uncancel"):
|
|
118
|
+
task.uncancel()
|
|
119
|
+
elif exc_type is not None and issubclass(exc_type, RunTerminatedError):
|
|
120
|
+
logger.warning("run was terminated by the platform; stopping")
|
|
121
|
+
self._state.terminated = True
|
|
122
|
+
suppress = True
|
|
123
|
+
elif exc_type is None:
|
|
124
|
+
await self._finalize(None)
|
|
125
|
+
elif issubclass(exc_type, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)):
|
|
126
|
+
# 外部取消(如本地 Ctrl-C 两次)/ 中断:按取消收尾
|
|
127
|
+
self._abort.ensure_reason(AbortReason.SIGNAL)
|
|
128
|
+
if self._finalizing is not None:
|
|
129
|
+
# 收尾已在途却没完(卡在慢写回 / 上报):第二次信号不再等它
|
|
130
|
+
self._finalizing.cancel()
|
|
131
|
+
else:
|
|
132
|
+
await self._finalize(None)
|
|
133
|
+
else:
|
|
134
|
+
# 平台拒收写回(push_data 把 PushRejectedError 抛给了作者)以拒收为准
|
|
135
|
+
await self._finalize(self._state.fatal or {
|
|
136
|
+
"code": "app-error", "category": "internal",
|
|
137
|
+
"message": f"{exc_type.__name__}: {exc}", "retryable": False})
|
|
138
|
+
failed_locally = (self._backend is not None
|
|
139
|
+
and not self._backend.reports_outcome
|
|
140
|
+
and self._outcome in ("failed", "partially_succeeded"))
|
|
141
|
+
finally:
|
|
142
|
+
await self._teardown()
|
|
143
|
+
if suppress and failed_locally:
|
|
144
|
+
# 本地 fail() 的退出码要非 0,方便脚本/CI 判断
|
|
145
|
+
raise SystemExit(1)
|
|
146
|
+
return suppress
|
|
147
|
+
|
|
148
|
+
async def _teardown(self) -> None:
|
|
149
|
+
ticker = self._ticker
|
|
150
|
+
if ticker is not None and ticker is not asyncio.current_task() and not ticker.done():
|
|
151
|
+
ticker.cancel()
|
|
152
|
+
try:
|
|
153
|
+
await ticker
|
|
154
|
+
except (asyncio.CancelledError, Exception):
|
|
155
|
+
pass
|
|
156
|
+
await self._abort.close()
|
|
157
|
+
# ABORTING 任务被 close() 取消时,它发起的收尾仍在跑:等它上报完终态再关后端
|
|
158
|
+
finalizing = self._finalizing
|
|
159
|
+
if finalizing is not None and not finalizing.done():
|
|
160
|
+
try:
|
|
161
|
+
await asyncio.shield(finalizing)
|
|
162
|
+
except asyncio.CancelledError:
|
|
163
|
+
if not finalizing.cancelled():
|
|
164
|
+
finalizing.cancel() # 主任务自己被取消:收尾也不再等
|
|
165
|
+
raise
|
|
166
|
+
# 收尾被主动放弃(第二次信号):静默
|
|
167
|
+
except Exception: # noqa: BLE001 — _do_finalize 内部已兜住,防御
|
|
168
|
+
pass
|
|
169
|
+
if self._backend is not None:
|
|
170
|
+
await self._backend.close()
|
|
171
|
+
self._reset()
|
|
172
|
+
|
|
173
|
+
# ── 作者 API ─────────────────────────────────────────────
|
|
174
|
+
async def get_input(self) -> dict[str, Any]:
|
|
175
|
+
"""Return the run input, with top-level defaults from input_schema.json applied."""
|
|
176
|
+
return self._require_info().input
|
|
177
|
+
|
|
178
|
+
def get_env(self, alias: str, default: str | None = None) -> str | None:
|
|
179
|
+
"""Return the value of an env alias declared in .dataapp/runtime.json.
|
|
180
|
+
|
|
181
|
+
Treat every value as a secret: never log it or push it as data.
|
|
182
|
+
"""
|
|
183
|
+
self._require_info()
|
|
184
|
+
assert self._backend is not None
|
|
185
|
+
value = self._backend.get_env(alias)
|
|
186
|
+
return default if value is None else value
|
|
187
|
+
|
|
188
|
+
async def push_data(self, data: dict[str, Any] | list[dict[str, Any]]) -> None:
|
|
189
|
+
"""Push one record or a list of records. Writes are batched by the SDK;
|
|
190
|
+
fields must match .dataapp/output_schema.json."""
|
|
191
|
+
info = self._require_info()
|
|
192
|
+
records = data if isinstance(data, list) else [data]
|
|
193
|
+
# 入口就序列化一遍:不可序列化的记录若进了队列,每次重发都在同一处炸,
|
|
194
|
+
# 堵死后面所有批次。整批先校验,有一条不合格就整批不收
|
|
195
|
+
sizes = [encoded_size(rec) for rec in records]
|
|
196
|
+
state = self._state
|
|
197
|
+
if state.fatal is not None:
|
|
198
|
+
raise DataAppError(f"the run is failing ({state.fatal['message']}); "
|
|
199
|
+
f"no more records are accepted")
|
|
200
|
+
if state.finished or state.terminated:
|
|
201
|
+
logger.warning("push_data after the run ended; %d record(s) dropped", len(records))
|
|
202
|
+
return
|
|
203
|
+
writer = self._require_writer()
|
|
204
|
+
if writer.capped:
|
|
205
|
+
return
|
|
206
|
+
await writer.wait_for_backlog()
|
|
207
|
+
records, sizes = writer.accept(records, sizes)
|
|
208
|
+
assert self._backend is not None
|
|
209
|
+
for rec in records:
|
|
210
|
+
self._backend.check_record(rec)
|
|
211
|
+
writer.add(records, sizes)
|
|
212
|
+
if writer.buffer_full() or writer.capped:
|
|
213
|
+
await writer.flush_soon()
|
|
214
|
+
if writer.capped:
|
|
215
|
+
logger.info("max_records (%s) reached; further records are dropped",
|
|
216
|
+
info.max_records)
|
|
217
|
+
self._abort.trigger(AbortReason.MAX_RECORDS)
|
|
218
|
+
|
|
219
|
+
async def set_progress(self, done: int | None = None, total: int | None = None,
|
|
220
|
+
status_text: str | None = None, *,
|
|
221
|
+
upstream_ref: str | None = None) -> None:
|
|
222
|
+
"""Report progress and an optional human-readable status. Throttled: only
|
|
223
|
+
the latest value is sent.
|
|
224
|
+
|
|
225
|
+
``upstream_ref`` is the upstream system's own task or job id (for example a
|
|
226
|
+
submission receipt). It is shown on the run so users can trace the run in
|
|
227
|
+
the upstream system. It is kept until a progress report succeeds, even
|
|
228
|
+
if later progress values replace the rest."""
|
|
229
|
+
self._require_info()
|
|
230
|
+
assert self._progress is not None
|
|
231
|
+
self._progress.set(done, total, status_text, upstream_ref)
|
|
232
|
+
if self._progress.due(time.monotonic()):
|
|
233
|
+
await self._progress.send()
|
|
234
|
+
|
|
235
|
+
def on(self, event: Event, handler: Handler) -> None:
|
|
236
|
+
"""Register an event handler (sync or async).
|
|
237
|
+
|
|
238
|
+
ABORTING fires once on cancel, timeout, record cap or SIGTERM/SIGINT,
|
|
239
|
+
with ``{"reason": ...}``. Without any ABORTING handler the SDK calls
|
|
240
|
+
:meth:`exit` itself.
|
|
241
|
+
"""
|
|
242
|
+
self._abort.on(event, handler)
|
|
243
|
+
|
|
244
|
+
async def exit(self) -> None:
|
|
245
|
+
"""Flush pending records, report the final state and end the
|
|
246
|
+
``async with DataApp`` block."""
|
|
247
|
+
self._require_info()
|
|
248
|
+
await self._finalize(None)
|
|
249
|
+
await self._stop_main()
|
|
250
|
+
|
|
251
|
+
async def fail(self, message: str, *, code: str = "app-failed",
|
|
252
|
+
retryable: bool = False) -> None:
|
|
253
|
+
"""End the run with a structured error. With ``retryable=True`` and no
|
|
254
|
+
records pushed yet, the platform may retry the run.
|
|
255
|
+
|
|
256
|
+
If the run is already finishing (for example the ``async with`` block
|
|
257
|
+
has just ended), the final state already being reported wins and this
|
|
258
|
+
error is ignored."""
|
|
259
|
+
self._require_info()
|
|
260
|
+
await self._finalize({"code": code, "category": "internal",
|
|
261
|
+
"message": message, "retryable": retryable})
|
|
262
|
+
await self._stop_main()
|
|
263
|
+
|
|
264
|
+
@property
|
|
265
|
+
def log(self) -> logging.Logger:
|
|
266
|
+
"""Logger writing to stderr; env values are masked in the output."""
|
|
267
|
+
return _log.get_logger()
|
|
268
|
+
|
|
269
|
+
@property
|
|
270
|
+
def is_at_home(self) -> bool:
|
|
271
|
+
"""True when running on the platform, False in local mode."""
|
|
272
|
+
return platform_env_present()
|
|
273
|
+
|
|
274
|
+
# ── 内部 ─────────────────────────────────────────────────
|
|
275
|
+
def _require_info(self) -> RunInfo:
|
|
276
|
+
if self._info is None:
|
|
277
|
+
raise RuntimeError("use DataApp inside `async with DataApp:`")
|
|
278
|
+
return self._info
|
|
279
|
+
|
|
280
|
+
def _require_writer(self) -> RecordWriter:
|
|
281
|
+
assert self._writer is not None
|
|
282
|
+
return self._writer
|
|
283
|
+
|
|
284
|
+
async def _force_stop(self) -> None:
|
|
285
|
+
"""超时宽限到点:上报终态并结束主任务(与 exit() 同路径,只是不等作者)。"""
|
|
286
|
+
await self._finalize(None)
|
|
287
|
+
await self._stop_main()
|
|
288
|
+
|
|
289
|
+
async def _stop_main(self) -> None:
|
|
290
|
+
task = self._main_task
|
|
291
|
+
if task is None or task.done() or self._exiting:
|
|
292
|
+
return # 主任务已在 __aexit__ 收尾:它自己会走完,不能再取消
|
|
293
|
+
self._stopping = True
|
|
294
|
+
task.cancel()
|
|
295
|
+
if asyncio.current_task() is task:
|
|
296
|
+
await asyncio.sleep(0) # 在主任务里调用:立即让 CancelledError 冒出来
|
|
297
|
+
|
|
298
|
+
async def _tick_loop(self) -> None:
|
|
299
|
+
"""后台节拍:攒批定时写出、进度节流补发、取消轮询、超时自检。"""
|
|
300
|
+
assert self._backend is not None and self._info is not None
|
|
301
|
+
assert self._progress is not None
|
|
302
|
+
writer = self._require_writer()
|
|
303
|
+
while not self._state.finished:
|
|
304
|
+
await asyncio.sleep(TICK)
|
|
305
|
+
now = time.monotonic()
|
|
306
|
+
try:
|
|
307
|
+
if writer.due(now):
|
|
308
|
+
await writer.flush()
|
|
309
|
+
if self._progress.due(now):
|
|
310
|
+
await self._progress.send()
|
|
311
|
+
if now - self._last_cancel_poll >= CANCEL_POLL_INTERVAL:
|
|
312
|
+
self._last_cancel_poll = now
|
|
313
|
+
if await self._backend.poll_cancel():
|
|
314
|
+
self._abort.request_cancel()
|
|
315
|
+
if self._info.timeout_at is not None and time.time() >= self._info.timeout_at:
|
|
316
|
+
self._abort.trigger(AbortReason.TIMEOUT)
|
|
317
|
+
except RunTerminatedError:
|
|
318
|
+
self._state.terminated = True
|
|
319
|
+
logger.warning("run was terminated by the platform; stopping")
|
|
320
|
+
await self._stop_main()
|
|
321
|
+
return
|
|
322
|
+
except PushRejectedError:
|
|
323
|
+
# 等同 fail(push-rejected):剩余批次照常写出,然后结束 with 块
|
|
324
|
+
await self._finalize(self._state.fatal)
|
|
325
|
+
await self._stop_main()
|
|
326
|
+
return
|
|
327
|
+
except Exception as exc: # noqa: BLE001 — 后台节拍出错不杀 run,下一拍再试
|
|
328
|
+
logger.warning("background sync failed: %s", exc)
|
|
329
|
+
|
|
330
|
+
async def _finalize(self, error: dict[str, Any] | None) -> None:
|
|
331
|
+
"""收尾单飞:第一个调用者定错误并起任务,之后的调用者(作者块体自然返回、
|
|
332
|
+
ABORTING 任务、节拍)都只等同一个任务。shield 住:等待方被取消不中断上报。"""
|
|
333
|
+
if self._finalizing is None:
|
|
334
|
+
state = self._state
|
|
335
|
+
if state.finished or self._backend is None:
|
|
336
|
+
return
|
|
337
|
+
state.finished = True # 先于一切 await 置位:push_data / 背压 / ABORTING 据此停下
|
|
338
|
+
self._finalizing = asyncio.create_task(self._do_finalize(error),
|
|
339
|
+
name="dataapp-finalize")
|
|
340
|
+
await asyncio.shield(self._finalizing)
|
|
341
|
+
|
|
342
|
+
async def _do_finalize(self, error: dict[str, Any] | None) -> None:
|
|
343
|
+
state = self._state
|
|
344
|
+
backend = self._backend
|
|
345
|
+
assert backend is not None
|
|
346
|
+
writer = self._require_writer()
|
|
347
|
+
assert self._progress is not None
|
|
348
|
+
flush_error: str | None = None
|
|
349
|
+
try:
|
|
350
|
+
await writer.drain()
|
|
351
|
+
await self._progress.send()
|
|
352
|
+
except RunTerminatedError:
|
|
353
|
+
state.terminated = True
|
|
354
|
+
except Exception as exc: # noqa: BLE001
|
|
355
|
+
logger.error("final flush failed: %s", exc)
|
|
356
|
+
flush_error = str(exc)
|
|
357
|
+
error = _outcome.merge_error(error, flush_error=flush_error, fatal=state.fatal)
|
|
358
|
+
|
|
359
|
+
abort = self._abort
|
|
360
|
+
if _outcome.needs_cancel_check(error, abort.reason, abort.cancel_requested):
|
|
361
|
+
try:
|
|
362
|
+
abort.cancel_requested = await backend.poll_cancel()
|
|
363
|
+
except Exception: # noqa: BLE001
|
|
364
|
+
pass
|
|
365
|
+
timeout_at = self._info.timeout_at if self._info is not None else None
|
|
366
|
+
result = _outcome.resolve(
|
|
367
|
+
error, reason=abort.reason, cancel_requested=abort.cancel_requested,
|
|
368
|
+
timeout_reached=timeout_at is not None and time.time() >= timeout_at,
|
|
369
|
+
confirmed=writer.confirmed)
|
|
370
|
+
self._outcome = result.state
|
|
371
|
+
if state.terminated:
|
|
372
|
+
return
|
|
373
|
+
try:
|
|
374
|
+
await backend.finish(result.state, result.partial, result.error)
|
|
375
|
+
except RunTerminatedError:
|
|
376
|
+
state.terminated = True
|
|
377
|
+
except Exception as exc: # noqa: BLE001 — 上报不到就交给平台按容器退出兜底
|
|
378
|
+
logger.error("failed to report final state: %s", exc)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
DataApp = _DataApp()
|
|
382
|
+
"""The process-wide DataApp instance."""
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""运行后端接口:平台(回连 run 级 API)与本地(读写 storage/)各一份实现。
|
|
2
|
+
|
|
3
|
+
DataApp 门面只依赖这里的接口,平台/本地差异全部收在后端里。
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any, Protocol
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class RunInfo:
|
|
14
|
+
input: dict[str, Any]
|
|
15
|
+
# runtime.env 的别名 -> 真实值;平台一律按敏感值处理
|
|
16
|
+
env: dict[str, str] = field(default_factory=dict)
|
|
17
|
+
# 平台层运行配置(独立于业务输入):写回条数上限,到点即熔断
|
|
18
|
+
max_records: int | None = None
|
|
19
|
+
# 由认领载荷的 timeout_seconds 派生(unix 秒):到点由 SDK 自己触发 ABORTING;平台不会
|
|
20
|
+
# 提前杀容器,只在执行环境生命期(远长于此)耗尽后收尸
|
|
21
|
+
timeout_at: float | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Backend(Protocol):
|
|
25
|
+
# True:终态经 finish() 上报给平台,进程退出码无关紧要;
|
|
26
|
+
# False(本地):失败要靠非 0 退出码让脚本/CI 感知
|
|
27
|
+
reports_outcome: bool
|
|
28
|
+
|
|
29
|
+
async def start(self) -> RunInfo: ...
|
|
30
|
+
|
|
31
|
+
async def write_records(self, records: list[dict[str, Any]], batch_id: str) -> int:
|
|
32
|
+
"""写出一批,返回平台已确认的该 run 总记录数。
|
|
33
|
+
|
|
34
|
+
可重试的失败抛普通异常;平台明确拒收抛 PushRejectedError。"""
|
|
35
|
+
...
|
|
36
|
+
|
|
37
|
+
async def report_progress(self, done: int | None, total: int | None,
|
|
38
|
+
status_text: str | None, upstream_ref: str | None = None) -> bool:
|
|
39
|
+
"""返回平台是否已请求取消(进度响应捎带取消信号)。"""
|
|
40
|
+
...
|
|
41
|
+
|
|
42
|
+
async def poll_cancel(self) -> bool: ...
|
|
43
|
+
|
|
44
|
+
async def finish(self, outcome: str, partial: bool,
|
|
45
|
+
error: dict[str, Any] | None) -> None: ...
|
|
46
|
+
|
|
47
|
+
def check_record(self, record: dict[str, Any]) -> None:
|
|
48
|
+
"""出参自检(只告警不拦截):本地对照 output_schema,平台侧由平台校验。"""
|
|
49
|
+
...
|
|
50
|
+
|
|
51
|
+
def get_env(self, alias: str) -> str | None: ...
|
|
52
|
+
|
|
53
|
+
async def close(self) -> None: ...
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""SDK 对作者可见的异常。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DataAppError(Exception):
|
|
7
|
+
"""SDK 无法继续:本地缺 input.json / 输入不合 schema / 平台拒绝本 SDK 版本等。"""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RunTerminatedError(Exception):
|
|
11
|
+
"""平台已把本 run 判为终态(超时强杀前后、被平台收尾),后续写回一律被拒。
|
|
12
|
+
|
|
13
|
+
SDK 在退出 `async with DataApp` 时静默吞掉它:run 的终态以平台为准。
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ProgressRejectedError(DataAppError):
|
|
18
|
+
"""平台明确拒收一次进度上报(4xx):同一载荷重发也不会成功,丢掉这次的进度值。"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PushRejectedError(DataAppError):
|
|
22
|
+
"""平台明确拒收某个写回批次(409 run-terminal 以外的 4xx):重发也不会成功。
|
|
23
|
+
|
|
24
|
+
这个批次被丢弃,run 以 push-rejected 失败收尾;此后的 push_data 一律报错。
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, message: str, code: str | None = None) -> None:
|
|
28
|
+
super().__init__(message)
|
|
29
|
+
self.code = code
|