octoparse-dataapp 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,24 @@
1
+ """octoparse-dataapp:开发 Octoparse DataHub Data App 的 SDK。
2
+
3
+ 进程归作者、SDK 是库:作者的 app.py 是入口,在 `async with DataApp:` 内
4
+ 读取输入、写回记录、响应取消。同一份代码本地与平台都用 `uv run python app.py`
5
+ 运行;存在 DATAHUB_SERVER_URL 与 DATAHUB_WORKER_TOKEN 即平台模式(按平台 worker 通道
6
+ 协议认领 run),否则本地模式。
7
+
8
+ 调用其他 Data App(发现、运行、取结果)属于用户侧能力,见 octoparse-client。
9
+ """
10
+
11
+ from importlib.metadata import PackageNotFoundError, version
12
+
13
+ from ._brand import BRAND
14
+
15
+ try:
16
+ __version__ = version(f"{BRAND}-dataapp")
17
+ except PackageNotFoundError: # 源码树直接导入(未安装)
18
+ __version__ = "0.0.0"
19
+
20
+ from ._app import AbortReason, DataApp, Event # noqa: E402
21
+ from ._errors import DataAppError, RunTerminatedError # noqa: E402
22
+
23
+ __all__ = ["DataApp", "Event", "AbortReason", "DataAppError", "RunTerminatedError",
24
+ "__version__"]
@@ -0,0 +1,119 @@
1
+ """ABORTING:取消 / 超时 / 条数熔断 / SIGTERM·SIGINT 统一走这里,只触发一次。
2
+
3
+ 有作者 handler 就交给作者收尾;没有就由 SDK 调 exit()。
4
+ 第二次信号不再等作者,直接取消主任务。
5
+ 超时(TIMEOUT)另有宽限:作者 handler 跑完后 TIMEOUT_GRACE_SECONDS 内 run 还没结束,
6
+ SDK 直接结束主任务——否则不配合收尾的作者代码会一直跑到执行环境生命期耗尽才被平台收尸。
7
+ 取消 / 熔断仍是协作式的,与引擎自己的看门狗同口径(只有超时强停)。
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import inspect
14
+ import signal
15
+ from collections.abc import Awaitable, Callable
16
+ from enum import Enum
17
+ from typing import Any
18
+
19
+ from ._log import sdk_logger as logger
20
+ from ._outcome import AbortReason
21
+ from ._state import RunState
22
+
23
+ # 超时后给作者收尾的宽限(秒):到点由 SDK 结束主任务;SDK 自己的攒批写出与终态上报在其后照常进行
24
+ TIMEOUT_GRACE_SECONDS = 15.0
25
+
26
+
27
+ class Event(str, Enum):
28
+ ABORTING = "aborting"
29
+
30
+
31
+ Handler = Callable[[dict[str, Any]], None | Awaitable[None]]
32
+
33
+
34
+ class AbortController:
35
+ def __init__(self, state: RunState, default_action: Callable[[], Awaitable[None]],
36
+ force_stop: Callable[[], Awaitable[None]] | None = None) -> None:
37
+ self._state = state
38
+ self._default_action = default_action # 没有 ABORTING handler 时的收尾(= exit())
39
+ self._force_stop = force_stop or default_action # 超时宽限到点的强停(= 结束主任务)
40
+ self._handlers: dict[Event, list[Handler]] = {}
41
+ self._task: asyncio.Task[None] | None = None
42
+ self._main_task: asyncio.Task[Any] | None = None
43
+ self._signals: list[int] = []
44
+ self.reason: AbortReason | None = None
45
+ self.cancel_requested = False
46
+
47
+ def on(self, event: Event, handler: Handler) -> None:
48
+ self._handlers.setdefault(Event(event), []).append(handler)
49
+
50
+ def trigger(self, reason: AbortReason) -> None:
51
+ if self.reason is not None or self._state.finished:
52
+ return
53
+ self.reason = reason
54
+ logger.info("aborting (%s)", reason.value)
55
+ self._task = asyncio.create_task(self._run_handlers(reason), name="dataapp-aborting")
56
+
57
+ def request_cancel(self) -> None:
58
+ """平台下发了取消信号(进度响应捎带或轮询得到)。"""
59
+ self.cancel_requested = True
60
+ self.trigger(AbortReason.CANCEL)
61
+
62
+ def ensure_reason(self, reason: AbortReason) -> None:
63
+ """只记原因不跑 handler:主任务已在收尾(外部取消、Ctrl-C)时用。"""
64
+ if self.reason is None:
65
+ self.reason = reason
66
+
67
+ async def _run_handlers(self, reason: AbortReason) -> None:
68
+ handlers = list(self._handlers.get(Event.ABORTING, ()))
69
+ if not handlers:
70
+ await self._default_action()
71
+ return
72
+ for handler in handlers:
73
+ try:
74
+ result = handler({"reason": reason.value})
75
+ if inspect.isawaitable(result):
76
+ await result
77
+ except asyncio.CancelledError:
78
+ raise
79
+ except Exception: # noqa: BLE001 — 作者代码
80
+ logger.exception("ABORTING handler failed")
81
+ if reason is AbortReason.TIMEOUT and not self._state.finished:
82
+ # 作者 handler 跑完却没结束 run:宽限后由 SDK 结束主任务,终态照常是 run-timeout
83
+ await asyncio.sleep(TIMEOUT_GRACE_SECONDS)
84
+ if not self._state.finished:
85
+ logger.warning("run did not stop within %.0fs after the timeout; stopping it",
86
+ TIMEOUT_GRACE_SECONDS)
87
+ await self._force_stop()
88
+
89
+ # ── 信号 ─────────────────────────────────────────────────
90
+ def install_signals(self, main_task: asyncio.Task[Any] | None) -> None:
91
+ self._main_task = main_task
92
+ loop = asyncio.get_running_loop()
93
+ for sig in (signal.SIGTERM, signal.SIGINT):
94
+ try:
95
+ loop.add_signal_handler(sig, self._on_signal)
96
+ except (NotImplementedError, RuntimeError, ValueError):
97
+ continue # Windows / 非主线程:没有信号处理也能跑,只是收不到 ABORTING
98
+ self._signals.append(sig)
99
+
100
+ def _on_signal(self) -> None:
101
+ if self.reason is not None and self._main_task is not None:
102
+ # 第二次信号:不再等作者收尾,直接取消主任务
103
+ self._main_task.cancel()
104
+ return
105
+ self.trigger(AbortReason.SIGNAL)
106
+
107
+ async def close(self) -> None:
108
+ loop = asyncio.get_running_loop()
109
+ for sig in self._signals:
110
+ loop.remove_signal_handler(sig)
111
+ self._signals.clear()
112
+ task = self._task
113
+ # 在 ABORTING 任务里调 exit() 走到这里时不能取消自己
114
+ if task is not None and task is not asyncio.current_task() and not task.done():
115
+ task.cancel()
116
+ try:
117
+ await task
118
+ except (asyncio.CancelledError, Exception):
119
+ pass
@@ -0,0 +1,382 @@
1
+ """DataApp 门面:进程归作者,SDK 是库。
2
+
3
+ async with DataApp: # 进入:拿到 run 与输入
4
+ inp = await DataApp.get_input()
5
+ await DataApp.push_data({...})
6
+ # 退出:上报终态(正常 = succeeded,异常 = failed)
7
+
8
+ 门面只做编排:写回管道在 _writer,ABORTING 与信号在 _abort,进度节流在 _progress,
9
+ 终态归约在 _outcome,平台/本地差异在后端。生命周期收尾(__aexit__ / exit() / fail()
10
+ 与主任务取消)刻意留在本文件:它们的正确性依赖 await 前后的置位顺序。
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import logging
17
+ import time
18
+ from pathlib import Path
19
+ from typing import Any
20
+
21
+ from . import _log, _outcome
22
+ from ._abort import AbortController, Event, Handler
23
+ from ._backend import Backend, RunInfo
24
+ from ._errors import DataAppError, PushRejectedError, RunTerminatedError
25
+ from ._local import LocalBackend
26
+ from ._outcome import AbortReason
27
+ from ._platform import (
28
+ ENV_PREFIX,
29
+ EXIT_OK,
30
+ EXIT_PLATFORM_UNAVAILABLE,
31
+ EXIT_USAGE,
32
+ NothingToRun,
33
+ PlatformBackend,
34
+ PlatformUnavailable,
35
+ platform_env_partial,
36
+ platform_env_present,
37
+ )
38
+ from ._progress import ProgressReporter
39
+ from ._state import RunState
40
+ from ._writer import RecordWriter, encoded_size
41
+
42
+ __all__ = ["AbortReason", "DataApp", "Event"]
43
+
44
+ logger = _log.sdk_logger
45
+
46
+ CANCEL_POLL_INTERVAL = 2.0 # 取消信号轮询(秒);平台侧兼作心跳
47
+ TICK = 0.25
48
+
49
+
50
+ class _DataApp:
51
+ def __init__(self) -> None:
52
+ self._reset()
53
+
54
+ def _reset(self) -> None:
55
+ self._state = RunState()
56
+ # 先于 async with 创建:作者可以在进入前就 on(ABORTING, ...)
57
+ self._abort = AbortController(self._state, self.exit, self._force_stop)
58
+ self._backend: Backend | None = None
59
+ self._info: RunInfo | None = None
60
+ self._writer: RecordWriter | None = None
61
+ self._progress: ProgressReporter | None = None
62
+ self._main_task: asyncio.Task[Any] | None = None
63
+ self._ticker: asyncio.Task[None] | None = None
64
+ self._stopping = False # exit()/fail() 主动结束了主任务
65
+ self._exiting = False # 主任务已进入 __aexit__:此后任何路径都不得再取消它
66
+ # 收尾单飞:谁先调 _finalize 谁创建这个任务,后来者只等它。独立成任务是为了
67
+ # 它不随发起方(ABORTING 任务 / 主任务 / 节拍)被取消而半途而废
68
+ self._finalizing: asyncio.Task[None] | None = None
69
+ self._outcome: str | None = None
70
+ self._last_cancel_poll = 0.0
71
+
72
+ # ── 生命周期 ─────────────────────────────────────────────
73
+ async def __aenter__(self) -> _DataApp:
74
+ if self._backend is not None:
75
+ raise RuntimeError("DataApp is already running")
76
+ _log.get_logger()
77
+ self._main_task = asyncio.current_task()
78
+ if platform_env_partial():
79
+ # 平台总是成对注入;缺一个只能是镜像 wrapper 吞掉了环境——不能悄悄落到本地模式
80
+ logger.error("incomplete platform environment: %sSERVER_URL and %sWORKER_TOKEN "
81
+ "must both be set", ENV_PREFIX, ENV_PREFIX)
82
+ raise SystemExit(EXIT_USAGE)
83
+ backend = self._backend = PlatformBackend() if platform_env_present() \
84
+ else LocalBackend(Path.cwd())
85
+ try:
86
+ self._info = await backend.start()
87
+ except BaseException as exc:
88
+ await backend.close()
89
+ self._reset()
90
+ # 认领阶段的结局按运行时协议的退出码交给平台归因;with 块体不执行
91
+ if isinstance(exc, NothingToRun):
92
+ logger.info("%s; exiting", exc)
93
+ raise SystemExit(EXIT_OK) from None
94
+ if isinstance(exc, PlatformUnavailable):
95
+ logger.error("platform unavailable: %s", exc)
96
+ raise SystemExit(EXIT_PLATFORM_UNAVAILABLE) from None
97
+ raise
98
+ self._writer = RecordWriter(backend, self._state, self._info.max_records)
99
+ self._progress = ProgressReporter(backend, self._state, self._abort.request_cancel)
100
+ _log.set_secrets(list(self._info.env.values()))
101
+ self._abort.install_signals(self._main_task)
102
+ self._ticker = asyncio.create_task(self._tick_loop(), name="dataapp-ticker")
103
+ return self
104
+
105
+ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
106
+ # 先于一切 await 置位:ABORTING 任务(熔断/取消/超时)可能恰在主任务收尾时
107
+ # 调 exit(),那时再取消主任务,CancelledError 会从 __aexit__ 里冒出去
108
+ self._exiting = True
109
+ suppress = False
110
+ failed_locally = False
111
+ try:
112
+ if self._stopping and exc_type is not None \
113
+ and issubclass(exc_type, asyncio.CancelledError):
114
+ # exit()/fail() 取消了主任务:这是预期的收尾路径
115
+ suppress = True
116
+ task = self._main_task
117
+ if task is not None and hasattr(task, "uncancel"):
118
+ task.uncancel()
119
+ elif exc_type is not None and issubclass(exc_type, RunTerminatedError):
120
+ logger.warning("run was terminated by the platform; stopping")
121
+ self._state.terminated = True
122
+ suppress = True
123
+ elif exc_type is None:
124
+ await self._finalize(None)
125
+ elif issubclass(exc_type, (asyncio.CancelledError, KeyboardInterrupt, SystemExit)):
126
+ # 外部取消(如本地 Ctrl-C 两次)/ 中断:按取消收尾
127
+ self._abort.ensure_reason(AbortReason.SIGNAL)
128
+ if self._finalizing is not None:
129
+ # 收尾已在途却没完(卡在慢写回 / 上报):第二次信号不再等它
130
+ self._finalizing.cancel()
131
+ else:
132
+ await self._finalize(None)
133
+ else:
134
+ # 平台拒收写回(push_data 把 PushRejectedError 抛给了作者)以拒收为准
135
+ await self._finalize(self._state.fatal or {
136
+ "code": "app-error", "category": "internal",
137
+ "message": f"{exc_type.__name__}: {exc}", "retryable": False})
138
+ failed_locally = (self._backend is not None
139
+ and not self._backend.reports_outcome
140
+ and self._outcome in ("failed", "partially_succeeded"))
141
+ finally:
142
+ await self._teardown()
143
+ if suppress and failed_locally:
144
+ # 本地 fail() 的退出码要非 0,方便脚本/CI 判断
145
+ raise SystemExit(1)
146
+ return suppress
147
+
148
+ async def _teardown(self) -> None:
149
+ ticker = self._ticker
150
+ if ticker is not None and ticker is not asyncio.current_task() and not ticker.done():
151
+ ticker.cancel()
152
+ try:
153
+ await ticker
154
+ except (asyncio.CancelledError, Exception):
155
+ pass
156
+ await self._abort.close()
157
+ # ABORTING 任务被 close() 取消时,它发起的收尾仍在跑:等它上报完终态再关后端
158
+ finalizing = self._finalizing
159
+ if finalizing is not None and not finalizing.done():
160
+ try:
161
+ await asyncio.shield(finalizing)
162
+ except asyncio.CancelledError:
163
+ if not finalizing.cancelled():
164
+ finalizing.cancel() # 主任务自己被取消:收尾也不再等
165
+ raise
166
+ # 收尾被主动放弃(第二次信号):静默
167
+ except Exception: # noqa: BLE001 — _do_finalize 内部已兜住,防御
168
+ pass
169
+ if self._backend is not None:
170
+ await self._backend.close()
171
+ self._reset()
172
+
173
+ # ── 作者 API ─────────────────────────────────────────────
174
+ async def get_input(self) -> dict[str, Any]:
175
+ """Return the run input, with top-level defaults from input_schema.json applied."""
176
+ return self._require_info().input
177
+
178
+ def get_env(self, alias: str, default: str | None = None) -> str | None:
179
+ """Return the value of an env alias declared in .dataapp/runtime.json.
180
+
181
+ Treat every value as a secret: never log it or push it as data.
182
+ """
183
+ self._require_info()
184
+ assert self._backend is not None
185
+ value = self._backend.get_env(alias)
186
+ return default if value is None else value
187
+
188
+ async def push_data(self, data: dict[str, Any] | list[dict[str, Any]]) -> None:
189
+ """Push one record or a list of records. Writes are batched by the SDK;
190
+ fields must match .dataapp/output_schema.json."""
191
+ info = self._require_info()
192
+ records = data if isinstance(data, list) else [data]
193
+ # 入口就序列化一遍:不可序列化的记录若进了队列,每次重发都在同一处炸,
194
+ # 堵死后面所有批次。整批先校验,有一条不合格就整批不收
195
+ sizes = [encoded_size(rec) for rec in records]
196
+ state = self._state
197
+ if state.fatal is not None:
198
+ raise DataAppError(f"the run is failing ({state.fatal['message']}); "
199
+ f"no more records are accepted")
200
+ if state.finished or state.terminated:
201
+ logger.warning("push_data after the run ended; %d record(s) dropped", len(records))
202
+ return
203
+ writer = self._require_writer()
204
+ if writer.capped:
205
+ return
206
+ await writer.wait_for_backlog()
207
+ records, sizes = writer.accept(records, sizes)
208
+ assert self._backend is not None
209
+ for rec in records:
210
+ self._backend.check_record(rec)
211
+ writer.add(records, sizes)
212
+ if writer.buffer_full() or writer.capped:
213
+ await writer.flush_soon()
214
+ if writer.capped:
215
+ logger.info("max_records (%s) reached; further records are dropped",
216
+ info.max_records)
217
+ self._abort.trigger(AbortReason.MAX_RECORDS)
218
+
219
+ async def set_progress(self, done: int | None = None, total: int | None = None,
220
+ status_text: str | None = None, *,
221
+ upstream_ref: str | None = None) -> None:
222
+ """Report progress and an optional human-readable status. Throttled: only
223
+ the latest value is sent.
224
+
225
+ ``upstream_ref`` is the upstream system's own task or job id (for example a
226
+ submission receipt). It is shown on the run so users can trace the run in
227
+ the upstream system. It is kept until a progress report succeeds, even
228
+ if later progress values replace the rest."""
229
+ self._require_info()
230
+ assert self._progress is not None
231
+ self._progress.set(done, total, status_text, upstream_ref)
232
+ if self._progress.due(time.monotonic()):
233
+ await self._progress.send()
234
+
235
+ def on(self, event: Event, handler: Handler) -> None:
236
+ """Register an event handler (sync or async).
237
+
238
+ ABORTING fires once on cancel, timeout, record cap or SIGTERM/SIGINT,
239
+ with ``{"reason": ...}``. Without any ABORTING handler the SDK calls
240
+ :meth:`exit` itself.
241
+ """
242
+ self._abort.on(event, handler)
243
+
244
+ async def exit(self) -> None:
245
+ """Flush pending records, report the final state and end the
246
+ ``async with DataApp`` block."""
247
+ self._require_info()
248
+ await self._finalize(None)
249
+ await self._stop_main()
250
+
251
+ async def fail(self, message: str, *, code: str = "app-failed",
252
+ retryable: bool = False) -> None:
253
+ """End the run with a structured error. With ``retryable=True`` and no
254
+ records pushed yet, the platform may retry the run.
255
+
256
+ If the run is already finishing (for example the ``async with`` block
257
+ has just ended), the final state already being reported wins and this
258
+ error is ignored."""
259
+ self._require_info()
260
+ await self._finalize({"code": code, "category": "internal",
261
+ "message": message, "retryable": retryable})
262
+ await self._stop_main()
263
+
264
+ @property
265
+ def log(self) -> logging.Logger:
266
+ """Logger writing to stderr; env values are masked in the output."""
267
+ return _log.get_logger()
268
+
269
+ @property
270
+ def is_at_home(self) -> bool:
271
+ """True when running on the platform, False in local mode."""
272
+ return platform_env_present()
273
+
274
+ # ── 内部 ─────────────────────────────────────────────────
275
+ def _require_info(self) -> RunInfo:
276
+ if self._info is None:
277
+ raise RuntimeError("use DataApp inside `async with DataApp:`")
278
+ return self._info
279
+
280
+ def _require_writer(self) -> RecordWriter:
281
+ assert self._writer is not None
282
+ return self._writer
283
+
284
+ async def _force_stop(self) -> None:
285
+ """超时宽限到点:上报终态并结束主任务(与 exit() 同路径,只是不等作者)。"""
286
+ await self._finalize(None)
287
+ await self._stop_main()
288
+
289
+ async def _stop_main(self) -> None:
290
+ task = self._main_task
291
+ if task is None or task.done() or self._exiting:
292
+ return # 主任务已在 __aexit__ 收尾:它自己会走完,不能再取消
293
+ self._stopping = True
294
+ task.cancel()
295
+ if asyncio.current_task() is task:
296
+ await asyncio.sleep(0) # 在主任务里调用:立即让 CancelledError 冒出来
297
+
298
+ async def _tick_loop(self) -> None:
299
+ """后台节拍:攒批定时写出、进度节流补发、取消轮询、超时自检。"""
300
+ assert self._backend is not None and self._info is not None
301
+ assert self._progress is not None
302
+ writer = self._require_writer()
303
+ while not self._state.finished:
304
+ await asyncio.sleep(TICK)
305
+ now = time.monotonic()
306
+ try:
307
+ if writer.due(now):
308
+ await writer.flush()
309
+ if self._progress.due(now):
310
+ await self._progress.send()
311
+ if now - self._last_cancel_poll >= CANCEL_POLL_INTERVAL:
312
+ self._last_cancel_poll = now
313
+ if await self._backend.poll_cancel():
314
+ self._abort.request_cancel()
315
+ if self._info.timeout_at is not None and time.time() >= self._info.timeout_at:
316
+ self._abort.trigger(AbortReason.TIMEOUT)
317
+ except RunTerminatedError:
318
+ self._state.terminated = True
319
+ logger.warning("run was terminated by the platform; stopping")
320
+ await self._stop_main()
321
+ return
322
+ except PushRejectedError:
323
+ # 等同 fail(push-rejected):剩余批次照常写出,然后结束 with 块
324
+ await self._finalize(self._state.fatal)
325
+ await self._stop_main()
326
+ return
327
+ except Exception as exc: # noqa: BLE001 — 后台节拍出错不杀 run,下一拍再试
328
+ logger.warning("background sync failed: %s", exc)
329
+
330
+ async def _finalize(self, error: dict[str, Any] | None) -> None:
331
+ """收尾单飞:第一个调用者定错误并起任务,之后的调用者(作者块体自然返回、
332
+ ABORTING 任务、节拍)都只等同一个任务。shield 住:等待方被取消不中断上报。"""
333
+ if self._finalizing is None:
334
+ state = self._state
335
+ if state.finished or self._backend is None:
336
+ return
337
+ state.finished = True # 先于一切 await 置位:push_data / 背压 / ABORTING 据此停下
338
+ self._finalizing = asyncio.create_task(self._do_finalize(error),
339
+ name="dataapp-finalize")
340
+ await asyncio.shield(self._finalizing)
341
+
342
+ async def _do_finalize(self, error: dict[str, Any] | None) -> None:
343
+ state = self._state
344
+ backend = self._backend
345
+ assert backend is not None
346
+ writer = self._require_writer()
347
+ assert self._progress is not None
348
+ flush_error: str | None = None
349
+ try:
350
+ await writer.drain()
351
+ await self._progress.send()
352
+ except RunTerminatedError:
353
+ state.terminated = True
354
+ except Exception as exc: # noqa: BLE001
355
+ logger.error("final flush failed: %s", exc)
356
+ flush_error = str(exc)
357
+ error = _outcome.merge_error(error, flush_error=flush_error, fatal=state.fatal)
358
+
359
+ abort = self._abort
360
+ if _outcome.needs_cancel_check(error, abort.reason, abort.cancel_requested):
361
+ try:
362
+ abort.cancel_requested = await backend.poll_cancel()
363
+ except Exception: # noqa: BLE001
364
+ pass
365
+ timeout_at = self._info.timeout_at if self._info is not None else None
366
+ result = _outcome.resolve(
367
+ error, reason=abort.reason, cancel_requested=abort.cancel_requested,
368
+ timeout_reached=timeout_at is not None and time.time() >= timeout_at,
369
+ confirmed=writer.confirmed)
370
+ self._outcome = result.state
371
+ if state.terminated:
372
+ return
373
+ try:
374
+ await backend.finish(result.state, result.partial, result.error)
375
+ except RunTerminatedError:
376
+ state.terminated = True
377
+ except Exception as exc: # noqa: BLE001 — 上报不到就交给平台按容器退出兜底
378
+ logger.error("failed to report final state: %s", exc)
379
+
380
+
381
+ DataApp = _DataApp()
382
+ """The process-wide DataApp instance."""
@@ -0,0 +1,53 @@
1
+ """运行后端接口:平台(回连 run 级 API)与本地(读写 storage/)各一份实现。
2
+
3
+ DataApp 门面只依赖这里的接口,平台/本地差异全部收在后端里。
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from dataclasses import dataclass, field
9
+ from typing import Any, Protocol
10
+
11
+
12
+ @dataclass
13
+ class RunInfo:
14
+ input: dict[str, Any]
15
+ # runtime.env 的别名 -> 真实值;平台一律按敏感值处理
16
+ env: dict[str, str] = field(default_factory=dict)
17
+ # 平台层运行配置(独立于业务输入):写回条数上限,到点即熔断
18
+ max_records: int | None = None
19
+ # 由认领载荷的 timeout_seconds 派生(unix 秒):到点由 SDK 自己触发 ABORTING;平台不会
20
+ # 提前杀容器,只在执行环境生命期(远长于此)耗尽后收尸
21
+ timeout_at: float | None = None
22
+
23
+
24
+ class Backend(Protocol):
25
+ # True:终态经 finish() 上报给平台,进程退出码无关紧要;
26
+ # False(本地):失败要靠非 0 退出码让脚本/CI 感知
27
+ reports_outcome: bool
28
+
29
+ async def start(self) -> RunInfo: ...
30
+
31
+ async def write_records(self, records: list[dict[str, Any]], batch_id: str) -> int:
32
+ """写出一批,返回平台已确认的该 run 总记录数。
33
+
34
+ 可重试的失败抛普通异常;平台明确拒收抛 PushRejectedError。"""
35
+ ...
36
+
37
+ async def report_progress(self, done: int | None, total: int | None,
38
+ status_text: str | None, upstream_ref: str | None = None) -> bool:
39
+ """返回平台是否已请求取消(进度响应捎带取消信号)。"""
40
+ ...
41
+
42
+ async def poll_cancel(self) -> bool: ...
43
+
44
+ async def finish(self, outcome: str, partial: bool,
45
+ error: dict[str, Any] | None) -> None: ...
46
+
47
+ def check_record(self, record: dict[str, Any]) -> None:
48
+ """出参自检(只告警不拦截):本地对照 output_schema,平台侧由平台校验。"""
49
+ ...
50
+
51
+ def get_env(self, alias: str) -> str | None: ...
52
+
53
+ async def close(self) -> None: ...
@@ -0,0 +1,4 @@
1
+ """Brand-specific constants, generated by scripts/brand-tree.sh for 'octoparse'.
2
+ The distribution name derives from BRAND; the env prefix and protocol header are brand-neutral."""
3
+
4
+ BRAND = "octoparse"
@@ -0,0 +1,29 @@
1
+ """SDK 对作者可见的异常。"""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class DataAppError(Exception):
7
+ """SDK 无法继续:本地缺 input.json / 输入不合 schema / 平台拒绝本 SDK 版本等。"""
8
+
9
+
10
+ class RunTerminatedError(Exception):
11
+ """平台已把本 run 判为终态(超时强杀前后、被平台收尾),后续写回一律被拒。
12
+
13
+ SDK 在退出 `async with DataApp` 时静默吞掉它:run 的终态以平台为准。
14
+ """
15
+
16
+
17
+ class ProgressRejectedError(DataAppError):
18
+ """平台明确拒收一次进度上报(4xx):同一载荷重发也不会成功,丢掉这次的进度值。"""
19
+
20
+
21
+ class PushRejectedError(DataAppError):
22
+ """平台明确拒收某个写回批次(409 run-terminal 以外的 4xx):重发也不会成功。
23
+
24
+ 这个批次被丢弃,run 以 push-rejected 失败收尾;此后的 push_data 一律报错。
25
+ """
26
+
27
+ def __init__(self, message: str, code: str | None = None) -> None:
28
+ super().__init__(message)
29
+ self.code = code