octoparse-dataapp 0.0.2__tar.gz

octoparse_dataapp-0.0.2/.gitignore

@@ -0,0 +1,27 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+
+# uv 虚拟环境
+.venv/
+uv.lock
+
+# 本地运行产物
+storage/
+
+# 工具缓存
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+
+# 本地环境变量（模板见 .env.example）
+.env
+
+# Claude Code 本地配置
+.claude/settings.local.json
+
+# OS
+.DS_Store

octoparse_dataapp-0.0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Octoparse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

octoparse_dataapp-0.0.2/PKG-INFO

@@ -0,0 +1,140 @@
+Metadata-Version: 2.5
+Name: octoparse-dataapp
+Version: 0.0.2
+Summary: Developer SDK for building Octoparse DataHub Data Apps: read input, push records, handle aborts
+Project-URL: Homepage, https://www.octoparse.com
+Author: Octoparse DataHub Team
+License: MIT
+License-File: LICENSE
+Keywords: data-app,datahub,octoparse,sdk
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: jsonschema>=4.0
+Description-Content-Type: text/markdown
+
+# octoparse-dataapp
+
+The developer SDK for building Octoparse DataHub Data Apps (`octoparse-dataapp`, imported as
+`octoparse_dataapp`). Your `app.py` is the program; the SDK is a library you call inside
+`async with DataApp:` to read the run input, push records and react to aborts.
+
+To discover, run and consume Data Apps as a caller, use `octoparse-client` instead.
+
+## Installation
+
+```bash
+uv add octoparse-dataapp
+```
+
+Start from the project template (`GET /v1/templates/python.zip`): it ships the `.dataapp/`
+contract, `app.py`, `Dockerfile` and `AGENTS.md`.
+
+## Usage
+
+```python
+import asyncio
+
+from octoparse_dataapp import DataApp, Event
+
+
+async def main() -> None:
+    async with DataApp:                        # enter: fetch the run input
+        inp = await DataApp.get_input()
+
+        async def on_abort(event) -> None:     # cancel / timeout / record cap
+            await DataApp.exit()
+        DataApp.on(Event.ABORTING, on_abort)
+
+        api_key = DataApp.get_env("example_api_key")   # alias from .dataapp/runtime.json
+        for i, keyword in enumerate(inp["keywords"], 1):
+            DataApp.log.info("processing %s", keyword)
+            await DataApp.push_data({"id": keyword, "title": f"Result for {keyword}"})
+            await DataApp.set_progress(i, len(inp["keywords"]))
+                                               # leave: report the final state
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Run it with `uv run python app.py`, both locally and on the platform.
+
+## API
+
+| API | Description |
+|---|---|
+| `async with DataApp:` | Starts the run. Leaving normally reports `succeeded`; an uncaught exception reports `failed` (or `partially_succeeded` if the platform has accepted records). |
+| `await DataApp.get_input()` | The run input, with top-level `default`s from `input_schema.json` applied. |
+| `await DataApp.push_data(record_or_list)` | Pushes records. Each record must be JSON-serializable (convert `datetime`, `Decimal`, `bytes`, NaN first) and match `output_schema.json`. The SDK batches writes and retries transient failures in the background; if the platform keeps refusing writes, `push_data` waits instead of buffering without limit. A batch the platform rejects outright fails the run with `push-rejected`. |
+| `DataApp.get_env(alias, default=None)` | Reads an env entry declared in `runtime.json` `env`. Treat values as secrets. |
+| `await DataApp.set_progress(done, total, status_text=None, *, upstream_ref=None)` | Progress and a human-readable status. Throttled; only the latest value is sent. `upstream_ref` is the upstream system's own task id, shown on the run for tracing. It is kept until a progress report succeeds, even if later progress values replace the rest. `status_text` longer than 500 characters is truncated. |
+| `DataApp.on(Event.ABORTING, handler)` | Called once on cancel, timeout, record cap or SIGTERM/SIGINT, with `{"reason": ...}`. Without a handler the SDK calls `exit()` for you. On timeout, if the run is still going 15 seconds after your handler returns, the SDK ends it (final state `run-timeout`); cancel stays cooperative. |
+| `await DataApp.exit()` | Flushes, reports the final state and ends the `async with` block. |
+| `await DataApp.fail(message, code=..., retryable=False)` | Ends the run with a structured error. |
+| `DataApp.log` | A `logging.Logger` writing to stderr; env values are masked. |
+| `DataApp.is_at_home` | `True` on the platform, `False` locally. |
+
+Custom billing quantities are derived by the platform from pushed records
+(`pricing.json` `events[].qty_from`); there is no manual charge call.
+
+## Local mode
+
+Without `DATAHUB_SERVER_URL` and `DATAHUB_WORKER_TOKEN` the SDK runs locally:
+
+- input is read from `storage/input.json` and validated against `.dataapp/input_schema.json`;
+- records are written to `storage/dataset/records.jsonl` (cleared on each run) and checked
+  against `.dataapp/output_schema.json` (warnings only);
+- `get_env` reads the process environment, then `.env` in the project root;
+- progress and the final state are logged; Ctrl-C triggers `ABORTING`, a second Ctrl-C
+  stops immediately; `fail()` exits with status 1.
+
+## Platform mode
+
+The platform runs your image as is (its own `ENTRYPOINT` / `CMD`, e.g. `python app.py`) and
+injects `DATAHUB_SERVER_URL` and `DATAHUB_WORKER_TOKEN` (a credential scoped to this one run).
+On entering `async with DataApp:` the SDK claims the run, then fetches the env entries
+declared in `runtime.json` if there are any. The run time limit comes from
+`execution.timeout_seconds`. Every request carries the runtime identity header
+`X-DataHub-Runtime: lang=python; sdk=<version>; protocol=1`.
+
+If the claim returns nothing to run (the run was cancelled before it started, or this SDK version is no
+longer supported and the platform has already failed the run), the process exits with status 0
+and the `async with` body never runs.
+
+Exit codes only matter if the process exits before it claims its run. The platform reads them as follows:
+
+| Code | Meaning |
+|---|---|
+| `0` | Nothing to run, or the claimed run already reached a final state |
+| `2` | Only one of `DATAHUB_SERVER_URL` / `DATAHUB_WORKER_TOKEN` is set (a wrapper dropped the environment) |
+| `75` | The platform could not be reached or rejected the credential; the platform retries the run |
+
+Do not exit with `75` from your own code: a pre-claim exit with that code is treated as a platform
+failure and retried.
+
+## Logs
+
+On the platform, everything your app writes to stdout and stderr (`DataApp.log`, `print`
+and uncaught errors) becomes the run's log. The person who started the run can read it, for
+test runs and production runs alike; the app owner cannot read the logs of runs started by
+others. The output of your image build is the build log, visible to you as the owner.
+
+Before a log is stored, the platform masks the env values handed to the run, inputs marked
+`sensitive`, platform credentials and common credential patterns. Anything else you print is
+shown as is, so do not print personal data or secrets you obtained in other ways. Run logs
+are kept for 30 days. A run keeps at most 2 MiB of log (the beginning and the end of each
+attempt), and lines longer than 16 KiB are cut.
+
+## License
+
+MIT

octoparse_dataapp-0.0.2/README.md

@@ -0,0 +1,116 @@
+# octoparse-dataapp
+
+The developer SDK for building Octoparse DataHub Data Apps (`octoparse-dataapp`, imported as
+`octoparse_dataapp`). Your `app.py` is the program; the SDK is a library you call inside
+`async with DataApp:` to read the run input, push records and react to aborts.
+
+To discover, run and consume Data Apps as a caller, use `octoparse-client` instead.
+
+## Installation
+
+```bash
+uv add octoparse-dataapp
+```
+
+Start from the project template (`GET /v1/templates/python.zip`): it ships the `.dataapp/`
+contract, `app.py`, `Dockerfile` and `AGENTS.md`.
+
+## Usage
+
+```python
+import asyncio
+
+from octoparse_dataapp import DataApp, Event
+
+
+async def main() -> None:
+    async with DataApp:                        # enter: fetch the run input
+        inp = await DataApp.get_input()
+
+        async def on_abort(event) -> None:     # cancel / timeout / record cap
+            await DataApp.exit()
+        DataApp.on(Event.ABORTING, on_abort)
+
+        api_key = DataApp.get_env("example_api_key")   # alias from .dataapp/runtime.json
+        for i, keyword in enumerate(inp["keywords"], 1):
+            DataApp.log.info("processing %s", keyword)
+            await DataApp.push_data({"id": keyword, "title": f"Result for {keyword}"})
+            await DataApp.set_progress(i, len(inp["keywords"]))
+                                               # leave: report the final state
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Run it with `uv run python app.py`, both locally and on the platform.
+
+## API
+
+| API | Description |
+|---|---|
+| `async with DataApp:` | Starts the run. Leaving normally reports `succeeded`; an uncaught exception reports `failed` (or `partially_succeeded` if the platform has accepted records). |
+| `await DataApp.get_input()` | The run input, with top-level `default`s from `input_schema.json` applied. |
+| `await DataApp.push_data(record_or_list)` | Pushes records. Each record must be JSON-serializable (convert `datetime`, `Decimal`, `bytes`, NaN first) and match `output_schema.json`. The SDK batches writes and retries transient failures in the background; if the platform keeps refusing writes, `push_data` waits instead of buffering without limit. A batch the platform rejects outright fails the run with `push-rejected`. |
+| `DataApp.get_env(alias, default=None)` | Reads an env entry declared in `runtime.json` `env`. Treat values as secrets. |
+| `await DataApp.set_progress(done, total, status_text=None, *, upstream_ref=None)` | Progress and a human-readable status. Throttled; only the latest value is sent. `upstream_ref` is the upstream system's own task id, shown on the run for tracing. It is kept until a progress report succeeds, even if later progress values replace the rest. `status_text` longer than 500 characters is truncated. |
+| `DataApp.on(Event.ABORTING, handler)` | Called once on cancel, timeout, record cap or SIGTERM/SIGINT, with `{"reason": ...}`. Without a handler the SDK calls `exit()` for you. On timeout, if the run is still going 15 seconds after your handler returns, the SDK ends it (final state `run-timeout`); cancel stays cooperative. |
+| `await DataApp.exit()` | Flushes, reports the final state and ends the `async with` block. |
+| `await DataApp.fail(message, code=..., retryable=False)` | Ends the run with a structured error. |
+| `DataApp.log` | A `logging.Logger` writing to stderr; env values are masked. |
+| `DataApp.is_at_home` | `True` on the platform, `False` locally. |
+
+Custom billing quantities are derived by the platform from pushed records
+(`pricing.json` `events[].qty_from`); there is no manual charge call.
+
+## Local mode
+
+Without `DATAHUB_SERVER_URL` and `DATAHUB_WORKER_TOKEN` the SDK runs locally:
+
+- input is read from `storage/input.json` and validated against `.dataapp/input_schema.json`;
+- records are written to `storage/dataset/records.jsonl` (cleared on each run) and checked
+  against `.dataapp/output_schema.json` (warnings only);
+- `get_env` reads the process environment, then `.env` in the project root;
+- progress and the final state are logged; Ctrl-C triggers `ABORTING`, a second Ctrl-C
+  stops immediately; `fail()` exits with status 1.
+
+## Platform mode
+
+The platform runs your image as is (its own `ENTRYPOINT` / `CMD`, e.g. `python app.py`) and
+injects `DATAHUB_SERVER_URL` and `DATAHUB_WORKER_TOKEN` (a credential scoped to this one run).
+On entering `async with DataApp:` the SDK claims the run, then fetches the env entries
+declared in `runtime.json` if there are any. The run time limit comes from
+`execution.timeout_seconds`. Every request carries the runtime identity header
+`X-DataHub-Runtime: lang=python; sdk=<version>; protocol=1`.
+
+If the claim returns nothing to run (the run was cancelled before it started, or this SDK version is no
+longer supported and the platform has already failed the run), the process exits with status 0
+and the `async with` body never runs.
+
+Exit codes only matter if the process exits before it claims its run. The platform reads them as follows:
+
+| Code | Meaning |
+|---|---|
+| `0` | Nothing to run, or the claimed run already reached a final state |
+| `2` | Only one of `DATAHUB_SERVER_URL` / `DATAHUB_WORKER_TOKEN` is set (a wrapper dropped the environment) |
+| `75` | The platform could not be reached or rejected the credential; the platform retries the run |
+
+Do not exit with `75` from your own code: a pre-claim exit with that code is treated as a platform
+failure and retried.
+
+## Logs
+
+On the platform, everything your app writes to stdout and stderr (`DataApp.log`, `print`
+and uncaught errors) becomes the run's log. The person who started the run can read it, for
+test runs and production runs alike; the app owner cannot read the logs of runs started by
+others. The output of your image build is the build log, visible to you as the owner.
+
+Before a log is stored, the platform masks the env values handed to the run, inputs marked
+`sensitive`, platform credentials and common credential patterns. Anything else you print is
+shown as is, so do not print personal data or secrets you obtained in other ways. Run logs
+are kept for 30 days. A run keeps at most 2 MiB of log (the beginning and the end of each
+attempt), and lines longer than 16 KiB are cut.
+
+## License
+
+MIT

octoparse_dataapp-0.0.2/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "octoparse-dataapp"
+version = "0.0.2"
+description = "Developer SDK for building Octoparse DataHub Data Apps: read input, push records, handle aborts"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Octoparse DataHub Team" }]
+keywords = ["octoparse", "datahub", "sdk", "data-app"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+]
+requires-python = ">=3.10"
+# 依赖克制、区间宽松：SDK 与作者依赖共存于同一 venv，收紧区间只会徒增
+# 与作者锁文件的冲突面。jsonschema 只在本地模式校验输入/输出时用到
+dependencies = ["httpx>=0.27", "jsonschema>=4.0"]
+
+[project.urls]
+Homepage = "https://www.octoparse.com"
+
+# 离线单元测试（tests/，本地模式走真实文件、平台模式走 MockTransport）：`uv run pytest -q`
+[dependency-groups]
+# pyyaml 只给 scripts/sync_contract.py 读引擎 openapi.yaml 用，不进运行时依赖
+dev = ["pytest>=8.0", "ruff>=0.6", "pyyaml>=6.0"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/octoparse_dataapp"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "A"]
+
+# sdist 只带公开内容：内部文档（AGENTS.md 等）不进公网制品
+[tool.hatch.build.targets.sdist]
+only-include = ["src", "README.md", "LICENSE"]

octoparse_dataapp-0.0.2/src/octoparse_dataapp/__init__.py

@@ -0,0 +1,24 @@
+"""octoparse-dataapp：开发 Octoparse DataHub Data App 的 SDK。
+
+进程归作者、SDK 是库：作者的 app.py 是入口，在 `async with DataApp:` 内
+读取输入、写回记录、响应取消。同一份代码本地与平台都用 `uv run python app.py`
+运行；存在 DATAHUB_SERVER_URL 与 DATAHUB_WORKER_TOKEN 即平台模式（按平台 worker 通道
+协议认领 run），否则本地模式。
+
+调用其他 Data App（发现、运行、取结果）属于用户侧能力，见 octoparse-client。
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from ._brand import BRAND
+
+try:
+    __version__ = version(f"{BRAND}-dataapp")
+except PackageNotFoundError:  # 源码树直接导入（未安装）
+    __version__ = "0.0.0"
+
+from ._app import AbortReason, DataApp, Event  # noqa: E402
+from ._errors import DataAppError, RunTerminatedError  # noqa: E402
+
+__all__ = ["DataApp", "Event", "AbortReason", "DataAppError", "RunTerminatedError",
+           "__version__"]

octoparse_dataapp-0.0.2/src/octoparse_dataapp/_abort.py

@@ -0,0 +1,119 @@
+"""ABORTING：取消 / 超时 / 条数熔断 / SIGTERM·SIGINT 统一走这里，只触发一次。
+
+有作者 handler 就交给作者收尾；没有就由 SDK 调 exit()。
+第二次信号不再等作者，直接取消主任务。
+超时（TIMEOUT）另有宽限：作者 handler 跑完后 TIMEOUT_GRACE_SECONDS 内 run 还没结束，
+SDK 直接结束主任务——否则不配合收尾的作者代码会一直跑到执行环境生命期耗尽才被平台收尸。
+取消 / 熔断仍是协作式的，与引擎自己的看门狗同口径（只有超时强停）。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import signal
+from collections.abc import Awaitable, Callable
+from enum import Enum
+from typing import Any
+
+from ._log import sdk_logger as logger
+from ._outcome import AbortReason
+from ._state import RunState
+
+# 超时后给作者收尾的宽限（秒）：到点由 SDK 结束主任务；SDK 自己的攒批写出与终态上报在其后照常进行
+TIMEOUT_GRACE_SECONDS = 15.0
+
+
+class Event(str, Enum):
+    ABORTING = "aborting"
+
+
+Handler = Callable[[dict[str, Any]], None | Awaitable[None]]
+
+
+class AbortController:
+    def __init__(self, state: RunState, default_action: Callable[[], Awaitable[None]],
+                 force_stop: Callable[[], Awaitable[None]] | None = None) -> None:
+        self._state = state
+        self._default_action = default_action   # 没有 ABORTING handler 时的收尾（= exit()）
+        self._force_stop = force_stop or default_action   # 超时宽限到点的强停（= 结束主任务）
+        self._handlers: dict[Event, list[Handler]] = {}
+        self._task: asyncio.Task[None] | None = None
+        self._main_task: asyncio.Task[Any] | None = None
+        self._signals: list[int] = []
+        self.reason: AbortReason | None = None
+        self.cancel_requested = False
+
+    def on(self, event: Event, handler: Handler) -> None:
+        self._handlers.setdefault(Event(event), []).append(handler)
+
+    def trigger(self, reason: AbortReason) -> None:
+        if self.reason is not None or self._state.finished:
+            return
+        self.reason = reason
+        logger.info("aborting (%s)", reason.value)
+        self._task = asyncio.create_task(self._run_handlers(reason), name="dataapp-aborting")
+
+    def request_cancel(self) -> None:
+        """平台下发了取消信号（进度响应捎带或轮询得到）。"""
+        self.cancel_requested = True
+        self.trigger(AbortReason.CANCEL)
+
+    def ensure_reason(self, reason: AbortReason) -> None:
+        """只记原因不跑 handler：主任务已在收尾（外部取消、Ctrl-C）时用。"""
+        if self.reason is None:
+            self.reason = reason
+
+    async def _run_handlers(self, reason: AbortReason) -> None:
+        handlers = list(self._handlers.get(Event.ABORTING, ()))
+        if not handlers:
+            await self._default_action()
+            return
+        for handler in handlers:
+            try:
+                result = handler({"reason": reason.value})
+                if inspect.isawaitable(result):
+                    await result
+            except asyncio.CancelledError:
+                raise
+            except Exception:  # noqa: BLE001 — 作者代码
+                logger.exception("ABORTING handler failed")
+        if reason is AbortReason.TIMEOUT and not self._state.finished:
+            # 作者 handler 跑完却没结束 run：宽限后由 SDK 结束主任务，终态照常是 run-timeout
+            await asyncio.sleep(TIMEOUT_GRACE_SECONDS)
+            if not self._state.finished:
+                logger.warning("run did not stop within %.0fs after the timeout; stopping it",
+                               TIMEOUT_GRACE_SECONDS)
+                await self._force_stop()
+
+    # ── 信号 ─────────────────────────────────────────────────
+    def install_signals(self, main_task: asyncio.Task[Any] | None) -> None:
+        self._main_task = main_task
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            try:
+                loop.add_signal_handler(sig, self._on_signal)
+            except (NotImplementedError, RuntimeError, ValueError):
+                continue   # Windows / 非主线程：没有信号处理也能跑，只是收不到 ABORTING
+            self._signals.append(sig)
+
+    def _on_signal(self) -> None:
+        if self.reason is not None and self._main_task is not None:
+            # 第二次信号：不再等作者收尾，直接取消主任务
+            self._main_task.cancel()
+            return
+        self.trigger(AbortReason.SIGNAL)
+
+    async def close(self) -> None:
+        loop = asyncio.get_running_loop()
+        for sig in self._signals:
+            loop.remove_signal_handler(sig)
+        self._signals.clear()
+        task = self._task
+        # 在 ABORTING 任务里调 exit() 走到这里时不能取消自己
+        if task is not None and task is not asyncio.current_task() and not task.done():
+            task.cancel()
+            try:
+                await task
+            except (asyncio.CancelledError, Exception):
+                pass