celestialflow 3.1.9__tar.gz → 3.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celestialflow-3.1.9 → celestialflow-3.2.1}/PKG-INFO +56 -29
- {celestialflow-3.1.9 → celestialflow-3.2.1}/README.md +54 -27
- {celestialflow-3.1.9 → celestialflow-3.2.1}/pyproject.toml +115 -81
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/__init__.py +21 -16
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/__init__.py +2 -2
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/core_inlet.py +22 -22
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/funnel/core_spout.py +20 -11
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/__init__.py +4 -4
- celestialflow-3.2.1/src/celestialflow/graph/core_graph.py +722 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/core_structure.py +104 -62
- celestialflow-3.2.1/src/celestialflow/graph/util_analysis.py +73 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/graph/util_serialize.py +40 -34
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/observability/__init__.py +6 -1
- celestialflow-3.2.1/src/celestialflow/observability/core_observer.py +67 -0
- celestialflow-3.2.1/src/celestialflow/observability/core_progress.py +61 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/observability/core_report.py +47 -73
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/__init__.py +2 -2
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_fail.py +50 -27
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_log.py +256 -117
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/core_success.py +2 -2
- celestialflow-3.2.1/src/celestialflow/persistence/util_jsonl.py +229 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/__init__.py +2 -2
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_dispatch.py +59 -46
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_envelope.py +15 -13
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/core_metrics.py +26 -23
- celestialflow-3.2.1/src/celestialflow/runtime/core_queue.py +253 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_errors.py +122 -24
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_estimators.py +12 -9
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/runtime/util_hash.py +10 -9
- celestialflow-3.2.1/src/celestialflow/runtime/util_types.py +217 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/__init__.py +4 -4
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_executor.py +207 -241
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_stage.py +73 -88
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/stage/core_stages.py +119 -74
- celestialflow-3.2.1/src/celestialflow/utils/__init__.py +5 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_benchmark.py +48 -34
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_clone.py +39 -30
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_collections.py +1 -1
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/utils/util_format.py +32 -25
- celestialflow-3.2.1/src/celestialflow/web/__init__.py +11 -0
- celestialflow-3.2.1/src/celestialflow/web/config.json +21 -0
- celestialflow-3.2.1/src/celestialflow/web/core_server.py +145 -0
- celestialflow-3.2.1/src/celestialflow/web/routes/__init__.py +33 -0
- celestialflow-3.2.1/src/celestialflow/web/routes/pull_routes.py +148 -0
- celestialflow-3.2.1/src/celestialflow/web/routes/push_routes.py +186 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/css/_colors.css +136 -135
- celestialflow-3.2.1/src/celestialflow/web/static/css/base.css +805 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard.css +71 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_analysis.css +39 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_history.css +87 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_statuses.css +238 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_structure.css +25 -0
- celestialflow-3.2.1/src/celestialflow/web/static/css/dashboard_summary.css +134 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/css/errors.css +30 -1
- celestialflow-3.1.9/src/celestialflow/web/static/css/inject.css → celestialflow-3.2.1/src/celestialflow/web/static/css/injection.css +57 -4
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_analysis.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_analysis.js +19 -12
- celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_history.js +356 -0
- celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_statuses.js +235 -0
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_structure.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_structure.js +23 -12
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_summary.js → celestialflow-3.2.1/src/celestialflow/web/static/js/dashboard_summary.js +20 -5
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_errors.js → celestialflow-3.2.1/src/celestialflow/web/static/js/errors.js +29 -19
- celestialflow-3.2.1/src/celestialflow/web/static/js/i18n.js +368 -0
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_injection.js → celestialflow-3.2.1/src/celestialflow/web/static/js/injection.js +101 -35
- celestialflow-3.2.1/src/celestialflow/web/static/js/layout_editor.js +158 -0
- celestialflow-3.2.1/src/celestialflow/web/static/js/main.js +235 -0
- celestialflow-3.2.1/src/celestialflow/web/static/js/utils.js +117 -0
- celestialflow-3.2.1/src/celestialflow/web/static/js/web_config.js +296 -0
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_analysis.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_analysis.ts +20 -17
- celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_history.ts +443 -0
- celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_statuses.ts +330 -0
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_structure.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_structure.ts +35 -16
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_summary.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/dashboard_summary.ts +25 -12
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_errors.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/errors.ts +33 -22
- celestialflow-3.2.1/src/celestialflow/web/static/ts/globals.d.ts +27 -0
- celestialflow-3.2.1/src/celestialflow/web/static/ts/i18n.ts +385 -0
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_injection.ts → celestialflow-3.2.1/src/celestialflow/web/static/ts/injection.ts +111 -41
- celestialflow-3.2.1/src/celestialflow/web/static/ts/layout_editor.ts +184 -0
- celestialflow-3.2.1/src/celestialflow/web/static/ts/main.ts +267 -0
- celestialflow-3.2.1/src/celestialflow/web/static/ts/utils.ts +132 -0
- celestialflow-3.2.1/src/celestialflow/web/static/ts/web_config.ts +345 -0
- celestialflow-3.2.1/src/celestialflow/web/templates/index.html +659 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_config.py +10 -6
- celestialflow-3.2.1/src/celestialflow/web/util_models.py +76 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/PKG-INFO +56 -29
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/SOURCES.txt +29 -28
- celestialflow-3.1.9/src/celestialflow/graph/core_graph.py +0 -895
- celestialflow-3.1.9/src/celestialflow/graph/util_analysis.py +0 -55
- celestialflow-3.1.9/src/celestialflow/observability/core_observer.py +0 -26
- celestialflow-3.1.9/src/celestialflow/observability/core_progress.py +0 -30
- celestialflow-3.1.9/src/celestialflow/persistence/util_jsonl.py +0 -183
- celestialflow-3.1.9/src/celestialflow/runtime/core_queue.py +0 -288
- celestialflow-3.1.9/src/celestialflow/runtime/util_factories.py +0 -95
- celestialflow-3.1.9/src/celestialflow/runtime/util_queue.py +0 -13
- celestialflow-3.1.9/src/celestialflow/runtime/util_types.py +0 -147
- celestialflow-3.1.9/src/celestialflow/utils/__init__.py +0 -1
- celestialflow-3.1.9/src/celestialflow/utils/util_debug.py +0 -16
- celestialflow-3.1.9/src/celestialflow/web/__init__.py +0 -6
- celestialflow-3.1.9/src/celestialflow/web/config.json +0 -35
- celestialflow-3.1.9/src/celestialflow/web/core_server.py +0 -522
- celestialflow-3.1.9/src/celestialflow/web/static/css/base.css +0 -320
- celestialflow-3.1.9/src/celestialflow/web/static/css/dashboard.css +0 -348
- celestialflow-3.1.9/src/celestialflow/web/static/js/main.js +0 -85
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_config.js +0 -124
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_history.js +0 -135
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_statuses.js +0 -121
- celestialflow-3.1.9/src/celestialflow/web/static/js/task_topology.js +0 -59
- celestialflow-3.1.9/src/celestialflow/web/static/js/utils.js +0 -269
- celestialflow-3.1.9/src/celestialflow/web/static/js/web_config.js +0 -119
- celestialflow-3.1.9/src/celestialflow/web/static/ts/globals.d.ts +0 -6
- celestialflow-3.1.9/src/celestialflow/web/static/ts/main.ts +0 -99
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_history.ts +0 -157
- celestialflow-3.1.9/src/celestialflow/web/static/ts/task_statuses.ts +0 -172
- celestialflow-3.1.9/src/celestialflow/web/static/ts/utils.ts +0 -314
- celestialflow-3.1.9/src/celestialflow/web/static/ts/web_config.ts +0 -144
- celestialflow-3.1.9/src/celestialflow/web/templates/index.html +0 -394
- celestialflow-3.1.9/tests/test_envelope.py +0 -51
- celestialflow-3.1.9/tests/test_executor.py +0 -382
- celestialflow-3.1.9/tests/test_graph.py +0 -579
- celestialflow-3.1.9/tests/test_metrics.py +0 -119
- celestialflow-3.1.9/tests/test_queue.py +0 -149
- celestialflow-3.1.9/tests/test_stage.py +0 -120
- celestialflow-3.1.9/tests/test_utils.py +0 -234
- {celestialflow-3.1.9 → celestialflow-3.2.1}/setup.cfg +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/persistence/util_constant.py +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/static/favicon.ico +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_cal.py +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow/web/util_error.py +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/dependency_links.txt +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/entry_points.txt +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/requires.txt +0 -0
- {celestialflow-3.1.9 → celestialflow-3.2.1}/src/celestialflow.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celestialflow
|
|
3
|
-
Version: 3.1
|
|
3
|
+
Version: 3.2.1
|
|
4
4
|
Summary: A flexible GRAPH-based task orchestration framework.
|
|
5
5
|
Author-email: Mr-xiaotian <mingxiaomingtian@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Classifier: Framework :: FastAPI
|
|
14
14
|
Classifier: Topic :: Software Development :: Libraries
|
|
15
|
-
Requires-Python: >=3.
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
Requires-Dist: tqdm
|
|
18
18
|
Requires-Dist: fastapi
|
|
@@ -44,7 +44,7 @@ Requires-Dist: celestialtree>=0.1.2
|
|
|
44
44
|
</p>
|
|
45
45
|
|
|
46
46
|
<p align="center">
|
|
47
|
-
<a href="README.md">中文</a> | <a href="docs/en/README.md">English</a> | <a href="docs/ja/README.md">日本語</a>
|
|
47
|
+
<a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/README.md">中文</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/en/README.md">English</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/ja/README.md">日本語</a>
|
|
48
48
|
</p>
|
|
49
49
|
|
|
50
50
|
**CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
|
|
@@ -64,11 +64,10 @@ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示
|
|
|
64
64
|
|
|
65
65
|
TaskStage 的任务执行模式同样包含三种,与TaskExecutor中一致。
|
|
66
66
|
|
|
67
|
-
在图级别上,每个 Stage
|
|
67
|
+
在图级别上,每个 Stage 支持两种上下文模式:
|
|
68
68
|
|
|
69
69
|
* **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
|
|
70
70
|
* **线程执行(thread layout)**:当前节点在主进程的独立线程中启动,适合 I/O 密集型任务和不可 pickle 的函数(如 lambda)。
|
|
71
|
-
* **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
|
|
72
71
|
|
|
73
72
|
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
|
|
74
73
|
|
|
@@ -150,16 +149,16 @@ def square(x):
|
|
|
150
149
|
|
|
151
150
|
if __name__ == "__main__":
|
|
152
151
|
# 定义两个任务节点
|
|
153
|
-
stage1 = TaskStage(name="Adder", func=add, execution_mode="thread", unpack_task_args=True
|
|
154
|
-
stage2 = TaskStage(name="Squarer"func=square,
|
|
152
|
+
stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
|
|
153
|
+
stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
|
|
155
154
|
|
|
156
155
|
# 构建任务图结构
|
|
157
156
|
graph = TaskGraph()
|
|
158
|
-
graph.set_stages(
|
|
157
|
+
graph.set_stages(stages=[stage1, stage2])
|
|
159
158
|
graph.connect([stage1], [stage2])
|
|
160
159
|
|
|
161
160
|
# 初始化任务并启动
|
|
162
|
-
graph.start_graph({stage1.
|
|
161
|
+
graph.start_graph({stage1.get_name(): [(1, 2), (3, 4), (5, 6)]})
|
|
163
162
|
```
|
|
164
163
|
|
|
165
164
|
注意不要在.ipynb中运行。
|
|
@@ -241,12 +240,12 @@ flowchart TD
|
|
|
241
240
|
|
|
242
241
|
## 环境要求(Requirements)
|
|
243
242
|
|
|
244
|
-
**CelestialFlow** 基于 Python 3.
|
|
243
|
+
**CelestialFlow** 基于 Python 3.11+,并依赖以下核心组件。
|
|
245
244
|
请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
|
|
246
245
|
|
|
247
246
|
| 依赖包 | 说明 |
|
|
248
247
|
| ----------------- | ---- |
|
|
249
|
-
| **Python ≥ 3.
|
|
248
|
+
| **Python ≥ 3.11** | 运行环境,建议使用 3.11 及以上版本 |
|
|
250
249
|
| **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
|
|
251
250
|
| **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
|
|
252
251
|
| **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
|
|
@@ -261,33 +260,61 @@ flowchart TD
|
|
|
261
260
|
<p align="center">
|
|
262
261
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
263
262
|
<br/>
|
|
264
|
-
<em>celestial-flow 3.1
|
|
263
|
+
<em>celestial-flow 3.2.1</em>
|
|
265
264
|
</p>
|
|
266
265
|
|
|
267
266
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
268
267
|
|
|
269
268
|
## 版本日志(Version Log)
|
|
270
|
-
- 3.1
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
-
|
|
283
|
-
|
|
284
|
-
|
|
269
|
+
- 3.2.1:
|
|
270
|
+
- feat:
|
|
271
|
+
- **[Important]** 大幅强化配置按钮功能
|
|
272
|
+
- 现在可以配置界面语言,并支持中、日、英三语言切换
|
|
273
|
+
- 可以配置错误日志每页日志条数
|
|
274
|
+
- 可以配置结构图中是否显示增量,不过现在样式上有点丑
|
|
275
|
+
- 以及最重要的,可以直接编辑仪表盘页的卡片布局
|
|
276
|
+
- 另外,所有设置保存成功后会有提示
|
|
277
|
+
- 折线图中可以选择数据类型
|
|
278
|
+
- 现在包括"数据累计"、"等待队列"和"数据变化率"
|
|
279
|
+
- 对 structure 的一些调整
|
|
280
|
+
- 所有 structure 的参数中添加 `stage_mode`,用于统一控制节点模式
|
|
281
|
+
- 对 struct 中节点不再重新命名
|
|
282
|
+
- 在部分 Spout 中增加缓冲机制
|
|
283
|
+
- refactor:
|
|
284
|
+
- **[Important]** 删除 executor 中的 `tag` 属性,并全面使用 `name` 属性进行替代
|
|
285
|
+
- `tag` 的出现是为了适应早期无法区分节点而设计的,但现在 `name` 已经强制唯一性,继续使用冗长的 `tag` 会导致不便
|
|
286
|
+
- **[Important]** 将 executor 中 `task_queue` 与 `result_queue` 的定义提前到 `__init__` 中
|
|
287
|
+
- 对于 executor 来说区别不大,但对于 stage 而言这是更理想且优雅的形式,只是由于原先 `stage_mode="process"` 时不允许执行函数内带有 `MPQueue`,所以才使用 graph 中统一定义,然后通过 process 的 args 注入的形式
|
|
288
|
+
- **[Important]** 重构 `core_server`,将其分解出 `routes/` 与 `util_models`,实现结构优化
|
|
289
|
+
- 进一步地,将 `log_queue` 和 `fail_queue` 的注入也提前到 `graph.set_stages` 中,实现 `stage.start_stage` 完全无输入参数
|
|
290
|
+
- 同时在 executor 中添加 `put_task` 与 `put_signal`,并在 graph 中进行复用
|
|
291
|
+
- 之前不能这样做是因为 stage 中此时还没有定义 queue
|
|
292
|
+
- 移除 `core_graph` 中的 `StageRuntime`
|
|
293
|
+
- 将类型标注中的部分 `Any` 改为更明确的类型
|
|
294
|
+
- 删除 HTML 中的仪表盘部分的 card,现在交给 `web-config` 来定义
|
|
295
|
+
- 删除部分没必要的前后端通信,优化通信负载
|
|
296
|
+
- 移除后端传递的节点历史信息,由前端进行维护
|
|
297
|
+
- 移除总体统计中除"总剩余时间"的后端数据,由前端进行计算
|
|
298
|
+
- 优化错误存储的 JSONL 数据结构,现在保存更细致的错误信息
|
|
299
|
+
- 调整 pyright 规则,并实现 pyright 0 error / 0 warning
|
|
300
|
+
- 给 `web/` 下几乎所有 `.ts` 与 `.css` 文件重命名,并整理文件内容边界
|
|
301
|
+
- `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
|
|
302
|
+
- 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
|
|
303
|
+
- 清简 `config.json` 中的部分字段
|
|
304
|
+
- 删除部分不必要的代码
|
|
305
|
+
- fix:
|
|
306
|
+
- 对部分太宽泛的错误捕捉进行收窄
|
|
285
307
|
- chore:
|
|
286
|
-
-
|
|
308
|
+
- 添加更多测试代码,并整理 `tests/` 结构
|
|
309
|
+
- 修复 `bench/` 中的一些问题
|
|
310
|
+
- 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
|
|
311
|
+
- 添加 `.github/workflows`,实现 CI/CD
|
|
312
|
+
- 更新支持的 Python 版本为 3.11,以与部分库要求相符
|
|
313
|
+
- 添加两个 skill,分别用于更新文档与审查项目
|
|
287
314
|
|
|
288
315
|
更多过往日志可看:
|
|
289
316
|
|
|
290
|
-
[change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/
|
|
317
|
+
[change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/change_log.md )
|
|
291
318
|
|
|
292
319
|
## Star 历史趋势(Star History)
|
|
293
320
|
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
</p>
|
|
20
20
|
|
|
21
21
|
<p align="center">
|
|
22
|
-
<a href="README.md">中文</a> | <a href="docs/en/README.md">English</a> | <a href="docs/ja/README.md">日本語</a>
|
|
22
|
+
<a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/README.md">中文</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/en/README.md">English</a> | <a href="https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/ja/README.md">日本語</a>
|
|
23
23
|
</p>
|
|
24
24
|
|
|
25
25
|
**CelestialFlow** 是一个轻量级但功能完全的任务流框架,适合需要 **复杂依赖关系**、**灵活执行模型**、**跨设备运行**与**实时可视化监控** 的中/大型 Python 任务系统。
|
|
@@ -39,11 +39,10 @@ TaskExecutor 实现了对任务的结果缓存,任务去重,进度条显示
|
|
|
39
39
|
|
|
40
40
|
TaskStage 的任务执行模式同样包含三种,与TaskExecutor中一致。
|
|
41
41
|
|
|
42
|
-
在图级别上,每个 Stage
|
|
42
|
+
在图级别上,每个 Stage 支持两种上下文模式:
|
|
43
43
|
|
|
44
44
|
* **线性执行(serial layout)**:当前节点执行完毕再启动下一节点(下游节点可提前接收任务但不会立即执行)。
|
|
45
45
|
* **线程执行(thread layout)**:当前节点在主进程的独立线程中启动,适合 I/O 密集型任务和不可 pickle 的函数(如 lambda)。
|
|
46
|
-
* **并行执行(process layout)**:当前节点启动后立刻前去启动下一节点。
|
|
47
46
|
|
|
48
47
|
TaskGraph 能构建完整的 **有向图结构(Directed Graph)**,不仅支持传统的有向无环图(DAG),也能灵活表达 **树形(Tree)**、**环形(loop)** 乃至于 **完全图(Complete Graph)** 形式的任务依赖。
|
|
49
48
|
|
|
@@ -125,16 +124,16 @@ def square(x):
|
|
|
125
124
|
|
|
126
125
|
if __name__ == "__main__":
|
|
127
126
|
# 定义两个任务节点
|
|
128
|
-
stage1 = TaskStage(name="Adder", func=add, execution_mode="thread", unpack_task_args=True
|
|
129
|
-
stage2 = TaskStage(name="Squarer"func=square,
|
|
127
|
+
stage1 = TaskStage(name="Adder", func=add, stage_mode="thread", execution_mode="thread", unpack_task_args=True)
|
|
128
|
+
stage2 = TaskStage(name="Squarer", func=square, stage_mode="thread", execution_mode="thread")
|
|
130
129
|
|
|
131
130
|
# 构建任务图结构
|
|
132
131
|
graph = TaskGraph()
|
|
133
|
-
graph.set_stages(
|
|
132
|
+
graph.set_stages(stages=[stage1, stage2])
|
|
134
133
|
graph.connect([stage1], [stage2])
|
|
135
134
|
|
|
136
135
|
# 初始化任务并启动
|
|
137
|
-
graph.start_graph({stage1.
|
|
136
|
+
graph.start_graph({stage1.get_name(): [(1, 2), (3, 4), (5, 6)]})
|
|
138
137
|
```
|
|
139
138
|
|
|
140
139
|
注意不要在.ipynb中运行。
|
|
@@ -216,12 +215,12 @@ flowchart TD
|
|
|
216
215
|
|
|
217
216
|
## 环境要求(Requirements)
|
|
218
217
|
|
|
219
|
-
**CelestialFlow** 基于 Python 3.
|
|
218
|
+
**CelestialFlow** 基于 Python 3.11+,并依赖以下核心组件。
|
|
220
219
|
请确保你的环境能够正常安装这些依赖(`pip install celestialflow` 会自动安装)。
|
|
221
220
|
|
|
222
221
|
| 依赖包 | 说明 |
|
|
223
222
|
| ----------------- | ---- |
|
|
224
|
-
| **Python ≥ 3.
|
|
223
|
+
| **Python ≥ 3.11** | 运行环境,建议使用 3.11 及以上版本 |
|
|
225
224
|
| **fastapi** | Web 服务接口框架(用于任务可视化与远程控制) |
|
|
226
225
|
| **uvicorn** | FastAPI 的高性能 ASGI 服务器 |
|
|
227
226
|
| **requests** | HTTP 客户端库,用于任务状态上报与远程调用 |
|
|
@@ -236,33 +235,61 @@ flowchart TD
|
|
|
236
235
|
<p align="center">
|
|
237
236
|
<img src="https://raw.githubusercontent.com/Mr-xiaotian/CelestialFlow/main/img/file_structure.svg" alt="FileStructure" />
|
|
238
237
|
<br/>
|
|
239
|
-
<em>celestial-flow 3.1
|
|
238
|
+
<em>celestial-flow 3.2.1</em>
|
|
240
239
|
</p>
|
|
241
240
|
|
|
242
241
|
(该视图由我的另一个项目[CelestialVault](https://github.com/Mr-xiaotian/CelestialVault)中inst_file.FileTree.print_tree()生成。转换为图片则借助[Carbon](https://carbon.now.sh)。)
|
|
243
242
|
|
|
244
243
|
## 版本日志(Version Log)
|
|
245
|
-
- 3.1
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
-
|
|
258
|
-
|
|
259
|
-
|
|
244
|
+
- 3.2.1:
|
|
245
|
+
- feat:
|
|
246
|
+
- **[Important]** 大幅强化配置按钮功能
|
|
247
|
+
- 现在可以配置界面语言,并支持中、日、英三语言切换
|
|
248
|
+
- 可以配置错误日志每页日志条数
|
|
249
|
+
- 可以配置结构图中是否显示增量,不过现在样式上有点丑
|
|
250
|
+
- 以及最重要的,可以直接编辑仪表盘页的卡片布局
|
|
251
|
+
- 另外,所有设置保存成功后会有提示
|
|
252
|
+
- 折线图中可以选择数据类型
|
|
253
|
+
- 现在包括"数据累计"、"等待队列"和"数据变化率"
|
|
254
|
+
- 对 structure 的一些调整
|
|
255
|
+
- 所有 structure 的参数中添加 `stage_mode`,用于统一控制节点模式
|
|
256
|
+
- 对 struct 中节点不再重新命名
|
|
257
|
+
- 在部分 Spout 中增加缓冲机制
|
|
258
|
+
- refactor:
|
|
259
|
+
- **[Important]** 删除 executor 中的 `tag` 属性,并全面使用 `name` 属性进行替代
|
|
260
|
+
- `tag` 的出现是为了适应早期无法区分节点而设计的,但现在 `name` 已经强制唯一性,继续使用冗长的 `tag` 会导致不便
|
|
261
|
+
- **[Important]** 将 executor 中 `task_queue` 与 `result_queue` 的定义提前到 `__init__` 中
|
|
262
|
+
- 对于 executor 来说区别不大,但对于 stage 而言这是更理想且优雅的形式,只是由于原先 `stage_mode="process"` 时不允许执行函数内带有 `MPQueue`,所以才使用 graph 中统一定义,然后通过 process 的 args 注入的形式
|
|
263
|
+
- **[Important]** 重构 `core_server`,将其分解出 `routes/` 与 `util_models`,实现结构优化
|
|
264
|
+
- 进一步地,将 `log_queue` 和 `fail_queue` 的注入也提前到 `graph.set_stages` 中,实现 `stage.start_stage` 完全无输入参数
|
|
265
|
+
- 同时在 executor 中添加 `put_task` 与 `put_signal`,并在 graph 中进行复用
|
|
266
|
+
- 之前不能这样做是因为 stage 中此时还没有定义 queue
|
|
267
|
+
- 移除 `core_graph` 中的 `StageRuntime`
|
|
268
|
+
- 将类型标注中的部分 `Any` 改为更明确的类型
|
|
269
|
+
- 删除 HTML 中的仪表盘部分的 card,现在交给 `web-config` 来定义
|
|
270
|
+
- 删除部分没必要的前后端通信,优化通信负载
|
|
271
|
+
- 移除后端传递的节点历史信息,由前端进行维护
|
|
272
|
+
- 移除总体统计中除"总剩余时间"的后端数据,由前端进行计算
|
|
273
|
+
- 优化错误存储的 JSONL 数据结构,现在保存更细致的错误信息
|
|
274
|
+
- 调整 pyright 规则,并实现 pyright 0 error / 0 warning
|
|
275
|
+
- 给 `web/` 下几乎所有 `.ts` 与 `.css` 文件重命名,并整理文件内容边界
|
|
276
|
+
- `util_error` 中添加更多错误类型,保证项目中所有主动 raise 的错误都在当前的错误体系下
|
|
277
|
+
- 删除前端代码中所有的 `onclick`,改用 `data-*` 属性
|
|
278
|
+
- 清简 `config.json` 中的部分字段
|
|
279
|
+
- 删除部分不必要的代码
|
|
280
|
+
- fix:
|
|
281
|
+
- 对部分太宽泛的错误捕捉进行收窄
|
|
260
282
|
- chore:
|
|
261
|
-
-
|
|
283
|
+
- 添加更多测试代码,并整理 `tests/` 结构
|
|
284
|
+
- 修复 `bench/` 中的一些问题
|
|
285
|
+
- 主要是 `bench_execution_mode` 中的两种 Fibonacci 运算量级不同,无法正常进行模型 bench
|
|
286
|
+
- 添加 `.github/workflows`,实现 CI/CD
|
|
287
|
+
- 更新支持的 Python 版本为 3.11,以与部分库要求相符
|
|
288
|
+
- 添加两个 skill,分别用于更新文档与审查项目
|
|
262
289
|
|
|
263
290
|
更多过往日志可看:
|
|
264
291
|
|
|
265
|
-
[change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/
|
|
292
|
+
[change_log.md](https://github.com/Mr-xiaotian/CelestialFlow/blob/main/docs/zh-CN/change_log.md )
|
|
266
293
|
|
|
267
294
|
## Star 历史趋势(Star History)
|
|
268
295
|
|
|
@@ -1,81 +1,115 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "celestialflow"
|
|
7
|
-
version = "3.1
|
|
8
|
-
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
license = { text = "MIT" }
|
|
11
|
-
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
-
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
-
requires-python = ">=3.
|
|
14
|
-
|
|
15
|
-
dependencies = [
|
|
16
|
-
"tqdm",
|
|
17
|
-
"fastapi",
|
|
18
|
-
"uvicorn",
|
|
19
|
-
"requests",
|
|
20
|
-
"networkx",
|
|
21
|
-
"redis",
|
|
22
|
-
"jinja2",
|
|
23
|
-
"celestialtree>=0.1.2",
|
|
24
|
-
]
|
|
25
|
-
|
|
26
|
-
classifiers = [
|
|
27
|
-
"Programming Language :: Python :: 3",
|
|
28
|
-
"License :: OSI Approved :: MIT License",
|
|
29
|
-
"Operating System :: OS Independent",
|
|
30
|
-
"Framework :: FastAPI",
|
|
31
|
-
"Topic :: Software Development :: Libraries",
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
[project.urls]
|
|
35
|
-
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
36
|
-
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
37
|
-
|
|
38
|
-
[project.scripts]
|
|
39
|
-
celestialflow-web = "celestialflow.web.core_server:main_entry"
|
|
40
|
-
|
|
41
|
-
[tool.setuptools]
|
|
42
|
-
license-files = []
|
|
43
|
-
|
|
44
|
-
[tool.setuptools.package-data]
|
|
45
|
-
celestialflow = [
|
|
46
|
-
"web/templates/**/*.html",
|
|
47
|
-
"web/static/**/*",
|
|
48
|
-
"web/config.json"
|
|
49
|
-
]
|
|
50
|
-
|
|
51
|
-
[tool.pytest.ini_options]
|
|
52
|
-
addopts = "-s"
|
|
53
|
-
log_cli = true
|
|
54
|
-
log_cli_level = "INFO"
|
|
55
|
-
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
56
|
-
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
57
|
-
|
|
58
|
-
filterwarnings = [
|
|
59
|
-
"ignore::DeprecationWarning",
|
|
60
|
-
"ignore::pytest.PytestDeprecationWarning",
|
|
61
|
-
]
|
|
62
|
-
|
|
63
|
-
asyncio_default_fixture_loop_scope = "function"
|
|
64
|
-
|
|
65
|
-
[dependency-groups]
|
|
66
|
-
dev = [
|
|
67
|
-
"build>=1.2.2.post1",
|
|
68
|
-
"twine>=6.1.0",
|
|
69
|
-
"pytest>=8.3.4",
|
|
70
|
-
"pytest-asyncio>=0.25.3",
|
|
71
|
-
"httpx>=0.28.1",
|
|
72
|
-
"python-dotenv>=1.2.2",
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
[
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "celestialflow"
|
|
7
|
+
version = "3.2.1"
|
|
8
|
+
description = "A flexible GRAPH-based task orchestration framework."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
authors = [{ name = "Mr-xiaotian", email = "mingxiaomingtian@gmail.com" }]
|
|
12
|
+
keywords = ["workflow", "task", "graph", "async", "CelestialFlow"]
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
|
|
15
|
+
dependencies = [
|
|
16
|
+
"tqdm",
|
|
17
|
+
"fastapi",
|
|
18
|
+
"uvicorn",
|
|
19
|
+
"requests",
|
|
20
|
+
"networkx",
|
|
21
|
+
"redis",
|
|
22
|
+
"jinja2",
|
|
23
|
+
"celestialtree>=0.1.2",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Framework :: FastAPI",
|
|
31
|
+
"Topic :: Software Development :: Libraries",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
"Homepage" = "https://github.com/Mr-xiaotian/CelestialFlow"
|
|
36
|
+
"Bug Tracker" = "https://github.com/Mr-xiaotian/CelestialFlow/issues"
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
celestialflow-web = "celestialflow.web.core_server:main_entry"
|
|
40
|
+
|
|
41
|
+
[tool.setuptools]
|
|
42
|
+
license-files = []
|
|
43
|
+
|
|
44
|
+
[tool.setuptools.package-data]
|
|
45
|
+
celestialflow = [
|
|
46
|
+
"web/templates/**/*.html",
|
|
47
|
+
"web/static/**/*",
|
|
48
|
+
"web/config.json"
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
addopts = "-s"
|
|
53
|
+
log_cli = true
|
|
54
|
+
log_cli_level = "INFO"
|
|
55
|
+
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
56
|
+
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
57
|
+
|
|
58
|
+
filterwarnings = [
|
|
59
|
+
"ignore::DeprecationWarning",
|
|
60
|
+
"ignore::pytest.PytestDeprecationWarning",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
64
|
+
|
|
65
|
+
[dependency-groups]
|
|
66
|
+
dev = [
|
|
67
|
+
"build>=1.2.2.post1",
|
|
68
|
+
"twine>=6.1.0",
|
|
69
|
+
"pytest>=8.3.4",
|
|
70
|
+
"pytest-asyncio>=0.25.3",
|
|
71
|
+
"httpx>=0.28.1",
|
|
72
|
+
"python-dotenv>=1.2.2",
|
|
73
|
+
"ruff>=0.15.0",
|
|
74
|
+
"pyright>=1.1.409",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[tool.pyright]
|
|
78
|
+
include = ["src"]
|
|
79
|
+
ignore = ["tests", "**/tests/**", "typings", "demo", "bench", "experiments", "examples", "temp"]
|
|
80
|
+
|
|
81
|
+
typeCheckingMode = "strict"
|
|
82
|
+
reportExplicitAny = "none"
|
|
83
|
+
reportAny = "none"
|
|
84
|
+
reportUnannotatedClassAttribute = "none"
|
|
85
|
+
reportUnusedFunction = "none"
|
|
86
|
+
reportImplicitOverride = "none"
|
|
87
|
+
|
|
88
|
+
[tool.ruff]
|
|
89
|
+
target-version = "py311"
|
|
90
|
+
exclude = ["tests", "typings", "demo", "bench", "experiments", "examples", "temp"]
|
|
91
|
+
|
|
92
|
+
[tool.ruff.lint]
|
|
93
|
+
select = [
|
|
94
|
+
"E", "F", # pycodestyle errors + pyflakes
|
|
95
|
+
"I", # isort (import order)
|
|
96
|
+
"UP", # pyupgrade
|
|
97
|
+
"B", # flake8-bugbear
|
|
98
|
+
"SIM", # flake8-simplify
|
|
99
|
+
"C4", # flake8-comprehensions
|
|
100
|
+
"RUF", # ruff-specific rules
|
|
101
|
+
]
|
|
102
|
+
ignore = [
|
|
103
|
+
"E501", # line too long(交给 formatter)
|
|
104
|
+
"RUF001", # 字符串内 Unicode 字符(中文内容)
|
|
105
|
+
"RUF002", # 中文标点(项目使用中文 docstring)
|
|
106
|
+
"RUF003", # 中文注释标点
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
[tool.ruff.format]
|
|
110
|
+
quote-style = "double"
|
|
111
|
+
indent-style = "space"
|
|
112
|
+
skip-magic-trailing-comma = false
|
|
113
|
+
|
|
114
|
+
[tool.ruff.lint.isort]
|
|
115
|
+
known-first-party = ["celestialflow"]
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
# __init__.py
|
|
2
|
+
"""CelestialFlow — 基于图结构的轻量级异步任务编排框架。
|
|
3
|
+
|
|
4
|
+
提供任务图构建、执行调度、实时监控、Web 可视化和持久化等核心能力。
|
|
5
|
+
"""
|
|
6
|
+
|
|
2
7
|
from .graph import (
|
|
3
8
|
TaskChain,
|
|
4
9
|
TaskComplete,
|
|
@@ -30,30 +35,30 @@ from .utils.util_format import format_table
|
|
|
30
35
|
from .web import TaskWebServer
|
|
31
36
|
|
|
32
37
|
__all__ = [
|
|
33
|
-
"
|
|
38
|
+
"BaseObserver",
|
|
39
|
+
"CallbackObserver",
|
|
34
40
|
"TaskChain",
|
|
35
|
-
"TaskLoop",
|
|
36
|
-
"TaskCross",
|
|
37
41
|
"TaskComplete",
|
|
38
|
-
"
|
|
42
|
+
"TaskCross",
|
|
43
|
+
"TaskExecutor",
|
|
44
|
+
"TaskGraph",
|
|
39
45
|
"TaskGrid",
|
|
40
|
-
"
|
|
41
|
-
"CallbackObserver",
|
|
46
|
+
"TaskLoop",
|
|
42
47
|
"TaskProgress",
|
|
43
|
-
"TaskExecutor",
|
|
44
|
-
"TaskStage",
|
|
45
|
-
"TaskSplitter",
|
|
46
|
-
"TaskRedisTransport",
|
|
47
|
-
"TaskRedisSource",
|
|
48
48
|
"TaskRedisAck",
|
|
49
|
+
"TaskRedisSource",
|
|
50
|
+
"TaskRedisTransport",
|
|
49
51
|
"TaskRouter",
|
|
50
|
-
"
|
|
52
|
+
"TaskSplitter",
|
|
53
|
+
"TaskStage",
|
|
51
54
|
"TaskWebServer",
|
|
55
|
+
"TaskWheel",
|
|
56
|
+
"TerminationSignal",
|
|
57
|
+
"benchmark_executor",
|
|
58
|
+
"benchmark_graph",
|
|
59
|
+
"format_table",
|
|
52
60
|
"load_jsonl_logs",
|
|
53
|
-
"load_task_by_stage",
|
|
54
61
|
"load_task_by_error",
|
|
62
|
+
"load_task_by_stage",
|
|
55
63
|
"make_hashable",
|
|
56
|
-
"format_table",
|
|
57
|
-
"benchmark_graph",
|
|
58
|
-
"benchmark_executor",
|
|
59
64
|
]
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
# funnel/core_inlet.py
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
from typing import Any
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class BaseInlet:
|
|
8
|
-
"""数据收集器基类,负责将记录通过队列发送到对应的监听器。"""
|
|
9
|
-
|
|
10
|
-
def __init__(self, queue: Any) -> None:
|
|
11
|
-
"""初始化收集器
|
|
12
|
-
|
|
13
|
-
:param queue: 记录队列
|
|
14
|
-
"""
|
|
15
|
-
self.queue: Any = queue
|
|
16
|
-
|
|
17
|
-
def _funnel(self, record: Any) -> None:
|
|
18
|
-
"""将记录放入队列
|
|
19
|
-
|
|
20
|
-
:param record: 待发送的记录
|
|
21
|
-
"""
|
|
22
|
-
self.queue.put(record)
|
|
1
|
+
# funnel/core_inlet.py
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BaseInlet:
|
|
8
|
+
"""数据收集器基类,负责将记录通过队列发送到对应的监听器。"""
|
|
9
|
+
|
|
10
|
+
def __init__(self, queue: Any) -> None:
|
|
11
|
+
"""初始化收集器
|
|
12
|
+
|
|
13
|
+
:param queue: 记录队列
|
|
14
|
+
"""
|
|
15
|
+
self.queue: Any = queue
|
|
16
|
+
|
|
17
|
+
def _funnel(self, record: Any) -> None:
|
|
18
|
+
"""将记录放入队列
|
|
19
|
+
|
|
20
|
+
:param record: 待发送的记录
|
|
21
|
+
"""
|
|
22
|
+
self.queue.put(record)
|