pyflowx 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyflowx-0.1.1/.github/workflows/ci.yml +138 -0
- pyflowx-0.1.1/.github/workflows/release.yml +193 -0
- pyflowx-0.1.1/.gitignore +11 -0
- pyflowx-0.1.1/.pre-commit-config.yaml +22 -0
- pyflowx-0.1.1/.python-version +1 -0
- pyflowx-0.1.1/.vscode/settings.json +36 -0
- pyflowx-0.1.1/PKG-INFO +274 -0
- pyflowx-0.1.1/README.md +241 -0
- pyflowx-0.1.1/pyproject.toml +89 -0
- pyflowx-0.1.1/src/pyflowx/__init__.py +75 -0
- pyflowx-0.1.1/src/pyflowx/__main__.py +9 -0
- pyflowx-0.1.1/src/pyflowx/context.py +194 -0
- pyflowx-0.1.1/src/pyflowx/errors.py +92 -0
- pyflowx-0.1.1/src/pyflowx/examples/__init__.py +0 -0
- pyflowx-0.1.1/src/pyflowx/examples/async_aggregation.py +58 -0
- pyflowx-0.1.1/src/pyflowx/examples/etl_pipeline.py +81 -0
- pyflowx-0.1.1/src/pyflowx/examples/parallel_run.py +59 -0
- pyflowx-0.1.1/src/pyflowx/executors.py +423 -0
- pyflowx-0.1.1/src/pyflowx/graph.py +242 -0
- pyflowx-0.1.1/src/pyflowx/py.typed +0 -0
- pyflowx-0.1.1/src/pyflowx/report.py +83 -0
- pyflowx-0.1.1/src/pyflowx/storage.py +133 -0
- pyflowx-0.1.1/src/pyflowx/task.py +144 -0
- pyflowx-0.1.1/tests/__init__.py +0 -0
- pyflowx-0.1.1/tests/test_context.py +235 -0
- pyflowx-0.1.1/tests/test_errors.py +87 -0
- pyflowx-0.1.1/tests/test_executors.py +515 -0
- pyflowx-0.1.1/tests/test_graph.py +230 -0
- pyflowx-0.1.1/tests/test_report.py +122 -0
- pyflowx-0.1.1/tests/test_storage.py +162 -0
- pyflowx-0.1.1/tests/test_task.py +63 -0
- pyflowx-0.1.1/tox.ini +21 -0
- pyflowx-0.1.1/uv.lock +3401 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, develop]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, develop]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
# ─────────────────────────────────────────────────────────────
|
|
16
|
+
# lint:代码风格与格式检查(单平台即可)
|
|
17
|
+
# ─────────────────────────────────────────────────────────────
|
|
18
|
+
lint:
|
|
19
|
+
name: Lint (ruff)
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout
|
|
23
|
+
uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- name: 安装 uv
|
|
26
|
+
uses: astral-sh/setup-uv@v5
|
|
27
|
+
with:
|
|
28
|
+
version: latest
|
|
29
|
+
enable-cache: true
|
|
30
|
+
cache-dependency-glob: uv.lock
|
|
31
|
+
|
|
32
|
+
- name: 设置 Python 3.13
|
|
33
|
+
uses: actions/setup-python@v5
|
|
34
|
+
with:
|
|
35
|
+
python-version: '3.13'
|
|
36
|
+
|
|
37
|
+
- name: 安装依赖
|
|
38
|
+
run: uv sync --extra dev --frozen
|
|
39
|
+
|
|
40
|
+
- name: Ruff 检查
|
|
41
|
+
run: uv run ruff check src tests examples
|
|
42
|
+
|
|
43
|
+
- name: Ruff 格式检查
|
|
44
|
+
run: uv run ruff format --check src tests examples
|
|
45
|
+
|
|
46
|
+
# ─────────────────────────────────────────────────────────────
|
|
47
|
+
# typecheck:mypy 严格类型检查
|
|
48
|
+
# ─────────────────────────────────────────────────────────────
|
|
49
|
+
typecheck:
|
|
50
|
+
name: Typecheck (mypy)
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
steps:
|
|
53
|
+
- name: Checkout
|
|
54
|
+
uses: actions/checkout@v4
|
|
55
|
+
|
|
56
|
+
- name: 安装 uv
|
|
57
|
+
uses: astral-sh/setup-uv@v5
|
|
58
|
+
with:
|
|
59
|
+
version: latest
|
|
60
|
+
enable-cache: true
|
|
61
|
+
cache-dependency-glob: uv.lock
|
|
62
|
+
|
|
63
|
+
- name: 设置 Python 3.13
|
|
64
|
+
uses: actions/setup-python@v5
|
|
65
|
+
with:
|
|
66
|
+
python-version: '3.13'
|
|
67
|
+
|
|
68
|
+
- name: 安装依赖
|
|
69
|
+
run: uv sync --extra dev --frozen
|
|
70
|
+
|
|
71
|
+
- name: Mypy 严格类型检查
|
|
72
|
+
run: uv run mypy
|
|
73
|
+
|
|
74
|
+
# ─────────────────────────────────────────────────────────────
|
|
75
|
+
# test:多平台 × 多 Python 版本矩阵测试 + 覆盖率
|
|
76
|
+
# ─────────────────────────────────────────────────────────────
|
|
77
|
+
test:
|
|
78
|
+
name: Test (${{ matrix.os }} / py${{ matrix.python-version }})
|
|
79
|
+
runs-on: ${{ matrix.os }}
|
|
80
|
+
strategy:
|
|
81
|
+
fail-fast: false
|
|
82
|
+
matrix:
|
|
83
|
+
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
84
|
+
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
|
|
85
|
+
steps:
|
|
86
|
+
- name: Checkout
|
|
87
|
+
uses: actions/checkout@v4
|
|
88
|
+
|
|
89
|
+
- name: 安装 uv
|
|
90
|
+
uses: astral-sh/setup-uv@v5
|
|
91
|
+
with:
|
|
92
|
+
version: latest
|
|
93
|
+
enable-cache: true
|
|
94
|
+
cache-dependency-glob: uv.lock
|
|
95
|
+
|
|
96
|
+
- name: 设置 Python ${{ matrix.python-version }}
|
|
97
|
+
uses: actions/setup-python@v5
|
|
98
|
+
with:
|
|
99
|
+
python-version: ${{ matrix.python-version }}
|
|
100
|
+
|
|
101
|
+
- name: 安装依赖
|
|
102
|
+
run: uv sync --extra dev --frozen
|
|
103
|
+
|
|
104
|
+
- name: 运行测试(含覆盖率,强制 100%)
|
|
105
|
+
run: uv run pytest -v --cov=pyflowx --cov-report=xml --cov-report=term-missing --cov-fail-under=100
|
|
106
|
+
|
|
107
|
+
- name: 运行示例冒烟测试
|
|
108
|
+
run: |
|
|
109
|
+
uv run python examples/etl_pipeline.py
|
|
110
|
+
uv run python examples/parallel_run.py
|
|
111
|
+
uv run python examples/async_aggregation.py
|
|
112
|
+
|
|
113
|
+
- name: 上传覆盖率
|
|
114
|
+
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
|
|
115
|
+
uses: actions/upload-artifact@v4
|
|
116
|
+
with:
|
|
117
|
+
name: coverage-${{ matrix.os }}-py${{ matrix.python-version }}
|
|
118
|
+
path: coverage.xml
|
|
119
|
+
retention-days: 7
|
|
120
|
+
|
|
121
|
+
# ─────────────────────────────────────────────────────────────
|
|
122
|
+
# 聚合:所有检查通过后才标记完成
|
|
123
|
+
# ─────────────────────────────────────────────────────────────
|
|
124
|
+
ci-pass:
|
|
125
|
+
name: CI Pass
|
|
126
|
+
runs-on: ubuntu-latest
|
|
127
|
+
needs: [lint, typecheck, test]
|
|
128
|
+
if: always()
|
|
129
|
+
steps:
|
|
130
|
+
- name: 检查依赖任务结果
|
|
131
|
+
if: ${{ needs.lint.result != 'success' || needs.typecheck.result != 'success' || needs.test.result != 'success' }}
|
|
132
|
+
run: |
|
|
133
|
+
echo "lint: ${{ needs.lint.result }}"
|
|
134
|
+
echo "typecheck: ${{ needs.typecheck.result }}"
|
|
135
|
+
echo "test: ${{ needs.test.result }}"
|
|
136
|
+
exit 1
|
|
137
|
+
- name: 全部通过
|
|
138
|
+
run: echo "✅ 所有 CI 检查通过"
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*.*.*'
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
inputs:
|
|
9
|
+
tag:
|
|
10
|
+
description: '发布版本号(如 v0.1.0)'
|
|
11
|
+
required: true
|
|
12
|
+
type: string
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: write
|
|
16
|
+
# Trusted Publishing (OIDC) 上传 PyPI 所需
|
|
17
|
+
id-token: write
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
# ─────────────────────────────────────────────────────────────
|
|
21
|
+
# 预检:版本号校验 + 与 pyproject.toml 一致性检查
|
|
22
|
+
# ─────────────────────────────────────────────────────────────
|
|
23
|
+
pre-check:
|
|
24
|
+
name: Pre-release Check
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
outputs:
|
|
27
|
+
version: ${{ steps.meta.outputs.version }}
|
|
28
|
+
tag: ${{ steps.meta.outputs.tag }}
|
|
29
|
+
steps:
|
|
30
|
+
- name: Checkout
|
|
31
|
+
uses: actions/checkout@v4
|
|
32
|
+
with:
|
|
33
|
+
fetch-depth: 0
|
|
34
|
+
|
|
35
|
+
- name: 解析版本号
|
|
36
|
+
id: meta
|
|
37
|
+
run: |
|
|
38
|
+
if [ -n "${{ inputs.tag }}" ]; then
|
|
39
|
+
TAG="${{ inputs.tag }}"
|
|
40
|
+
else
|
|
41
|
+
TAG="${GITHUB_REF#refs/tags/}"
|
|
42
|
+
fi
|
|
43
|
+
# 去除前缀 v
|
|
44
|
+
VERSION="${TAG#v}"
|
|
45
|
+
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
46
|
+
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
47
|
+
echo "发布版本: $VERSION (tag: $TAG)"
|
|
48
|
+
|
|
49
|
+
- name: 校验版本号格式
|
|
50
|
+
run: |
|
|
51
|
+
VERSION="${{ steps.meta.outputs.version }}"
|
|
52
|
+
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
|
53
|
+
echo "❌ 版本号格式错误: $VERSION(应为 x.y.z 或 x.y.z-rc.n)"
|
|
54
|
+
exit 1
|
|
55
|
+
fi
|
|
56
|
+
|
|
57
|
+
- name: 校验 pyproject.toml 版本一致
|
|
58
|
+
run: |
|
|
59
|
+
# 精确提取 [project] 段的 version 字段(避免匹配到依赖的 version)
|
|
60
|
+
PY_VERSION=$(awk '/^\[project\]/{f=1} f&&/^version[[:space:]]*=/{gsub(/[" ]/,"",$3); print $3; exit}' pyproject.toml)
|
|
61
|
+
echo "pyproject.toml version: $PY_VERSION"
|
|
62
|
+
if [ "$PY_VERSION" != "${{ steps.meta.outputs.version }}" ]; then
|
|
63
|
+
echo "❌ pyproject.toml 版本($PY_VERSION) 与 tag 版本(${{ steps.meta.outputs.version }}) 不一致"
|
|
64
|
+
echo "请先更新 pyproject.toml 中的 version 字段"
|
|
65
|
+
exit 1
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
# ─────────────────────────────────────────────────────────────
|
|
69
|
+
# 构建:wheel + sdist(纯 Python,单平台即可)
|
|
70
|
+
# ─────────────────────────────────────────────────────────────
|
|
71
|
+
build:
|
|
72
|
+
name: Build Artifacts
|
|
73
|
+
needs: pre-check
|
|
74
|
+
runs-on: ubuntu-latest
|
|
75
|
+
steps:
|
|
76
|
+
- name: Checkout
|
|
77
|
+
uses: actions/checkout@v4
|
|
78
|
+
|
|
79
|
+
- name: 安装 uv
|
|
80
|
+
uses: astral-sh/setup-uv@v5
|
|
81
|
+
with:
|
|
82
|
+
version: latest
|
|
83
|
+
enable-cache: true
|
|
84
|
+
|
|
85
|
+
- name: 设置 Python 3.13
|
|
86
|
+
uses: actions/setup-python@v5
|
|
87
|
+
with:
|
|
88
|
+
python-version: '3.13'
|
|
89
|
+
|
|
90
|
+
- name: 安装依赖
|
|
91
|
+
run: uv sync --extra dev --frozen
|
|
92
|
+
|
|
93
|
+
- name: 构建 wheel + sdist
|
|
94
|
+
run: uv build
|
|
95
|
+
|
|
96
|
+
- name: 校验产物
|
|
97
|
+
run: |
|
|
98
|
+
echo "待上传产物:"
|
|
99
|
+
ls -la dist/
|
|
100
|
+
if [ -z "$(ls -A dist/*.whl dist/*.tar.gz 2>/dev/null)" ]; then
|
|
101
|
+
echo "❌ 未找到 wheel 或 sdist 产物"
|
|
102
|
+
exit 1
|
|
103
|
+
fi
|
|
104
|
+
|
|
105
|
+
- name: 上传构建产物
|
|
106
|
+
uses: actions/upload-artifact@v4
|
|
107
|
+
with:
|
|
108
|
+
name: dist
|
|
109
|
+
path: dist/*
|
|
110
|
+
retention-days: 30
|
|
111
|
+
|
|
112
|
+
# ─────────────────────────────────────────────────────────────
|
|
113
|
+
# 发布:上传到 PyPI(Trusted Publishing / OIDC)
|
|
114
|
+
# ─────────────────────────────────────────────────────────────
|
|
115
|
+
publish-pypi:
|
|
116
|
+
name: Publish to PyPI
|
|
117
|
+
needs: [pre-check, build]
|
|
118
|
+
runs-on: ubuntu-latest
|
|
119
|
+
environment:
|
|
120
|
+
name: pypi
|
|
121
|
+
url: https://pypi.org/project/pyflowx/${{ needs.pre-check.outputs.version }}
|
|
122
|
+
permissions:
|
|
123
|
+
id-token: write
|
|
124
|
+
steps:
|
|
125
|
+
- name: 下载构建产物
|
|
126
|
+
uses: actions/download-artifact@v4
|
|
127
|
+
with:
|
|
128
|
+
name: dist
|
|
129
|
+
path: dist
|
|
130
|
+
|
|
131
|
+
- name: 上传到 PyPI
|
|
132
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
133
|
+
with:
|
|
134
|
+
attestations: true
|
|
135
|
+
|
|
136
|
+
# ─────────────────────────────────────────────────────────────
|
|
137
|
+
# 发布:创建 GitHub Release
|
|
138
|
+
# ─────────────────────────────────────────────────────────────
|
|
139
|
+
release:
|
|
140
|
+
name: Publish Release
|
|
141
|
+
needs: [pre-check, build, publish-pypi]
|
|
142
|
+
runs-on: ubuntu-latest
|
|
143
|
+
permissions:
|
|
144
|
+
contents: write
|
|
145
|
+
steps:
|
|
146
|
+
- name: Checkout
|
|
147
|
+
uses: actions/checkout@v4
|
|
148
|
+
|
|
149
|
+
- name: 下载构建产物
|
|
150
|
+
uses: actions/download-artifact@v4
|
|
151
|
+
with:
|
|
152
|
+
name: dist
|
|
153
|
+
path: assets
|
|
154
|
+
|
|
155
|
+
- name: 整理发布产物
|
|
156
|
+
run: |
|
|
157
|
+
ls -la assets/
|
|
158
|
+
|
|
159
|
+
- name: 生成 Release Notes
|
|
160
|
+
id: notes
|
|
161
|
+
run: |
|
|
162
|
+
{
|
|
163
|
+
echo "## pyflowx ${{ needs.pre-check.outputs.version }}"
|
|
164
|
+
echo ""
|
|
165
|
+
echo "### 下载"
|
|
166
|
+
echo ""
|
|
167
|
+
echo "- **Wheel**: \`pyflowx-${{ needs.pre-check.outputs.version }}-py3-none-any.whl\`"
|
|
168
|
+
echo "- **源码包**: \`pyflowx-${{ needs.pre-check.outputs.version }}.tar.gz\`"
|
|
169
|
+
echo ""
|
|
170
|
+
echo "### 安装"
|
|
171
|
+
echo ""
|
|
172
|
+
echo '```bash'
|
|
173
|
+
echo "pip install pyflowx==${{ needs.pre-check.outputs.version }}"
|
|
174
|
+
echo '```'
|
|
175
|
+
echo ""
|
|
176
|
+
echo "### 完整变更日志"
|
|
177
|
+
} > RELEASE_NOTES.md
|
|
178
|
+
{
|
|
179
|
+
echo "content<<EOF"
|
|
180
|
+
cat RELEASE_NOTES.md
|
|
181
|
+
echo "EOF"
|
|
182
|
+
} >> $GITHUB_OUTPUT
|
|
183
|
+
|
|
184
|
+
- name: 创建 GitHub Release
|
|
185
|
+
uses: softprops/action-gh-release@v2
|
|
186
|
+
with:
|
|
187
|
+
tag_name: ${{ needs.pre-check.outputs.tag }}
|
|
188
|
+
name: pyflowx ${{ needs.pre-check.outputs.version }}
|
|
189
|
+
body: ${{ steps.notes.outputs.content }}
|
|
190
|
+
files: assets/*
|
|
191
|
+
draft: false
|
|
192
|
+
prerelease: ${{ contains(needs.pre-check.outputs.version, '-') }}
|
|
193
|
+
generate_release_notes: true
|
pyflowx-0.1.1/.gitignore
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# prek compatible configuration
|
|
2
|
+
# See https://pre-commit.com for more information
|
|
3
|
+
repos:
|
|
4
|
+
- repo: https://gitcode.com/gh_mirrors/ru/ruff-pre-commit.git
|
|
5
|
+
# Ruff version - keep in sync with pyproject.toml
|
|
6
|
+
rev: v0.15.4
|
|
7
|
+
hooks:
|
|
8
|
+
# Run the linter
|
|
9
|
+
- id: ruff
|
|
10
|
+
args: [ --fix, --exit-non-zero-on-fix ]
|
|
11
|
+
# Run the formatter
|
|
12
|
+
- id: ruff-format
|
|
13
|
+
args: [ --config=pyproject.toml]
|
|
14
|
+
- repo: https://gitcode.com/gh_mirrors/pr/pre-commit-hooks.git
|
|
15
|
+
rev: v5.0.0
|
|
16
|
+
hooks:
|
|
17
|
+
- id: check-merge-conflict
|
|
18
|
+
- id: debug-statements
|
|
19
|
+
- id: fix-byte-order-marker
|
|
20
|
+
- id: trailing-whitespace
|
|
21
|
+
args: [ --markdown-linebreak-ext=md ]
|
|
22
|
+
- id: end-of-file-fixer
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.8
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"[python]": {
|
|
3
|
+
"editor.codeActionsOnSave": {
|
|
4
|
+
"source.fixAll.ruff": "always",
|
|
5
|
+
"source.organizeImports.ruff": "always",
|
|
6
|
+
"source.sort.json": "always"
|
|
7
|
+
},
|
|
8
|
+
"editor.defaultFormatter": "charliermarsh.ruff",
|
|
9
|
+
"editor.formatOnSave": true
|
|
10
|
+
},
|
|
11
|
+
"[toml]": {
|
|
12
|
+
"editor.defaultFormatter": "tamasfe.even-better-toml",
|
|
13
|
+
"editor.formatOnSave": true
|
|
14
|
+
},
|
|
15
|
+
"evenBetterToml.formatter.alignComments": true,
|
|
16
|
+
"evenBetterToml.formatter.alignEntries": true,
|
|
17
|
+
"evenBetterToml.formatter.allowedBlankLines": 1,
|
|
18
|
+
"evenBetterToml.formatter.arrayAutoCollapse": true,
|
|
19
|
+
"evenBetterToml.formatter.arrayAutoExpand": true,
|
|
20
|
+
"evenBetterToml.formatter.arrayTrailingComma": true,
|
|
21
|
+
"evenBetterToml.formatter.columnWidth": 120,
|
|
22
|
+
"evenBetterToml.formatter.compactEntries": false,
|
|
23
|
+
"evenBetterToml.formatter.indentEntries": false,
|
|
24
|
+
"evenBetterToml.formatter.indentTables": false,
|
|
25
|
+
"evenBetterToml.formatter.reorderArrays": true,
|
|
26
|
+
"evenBetterToml.formatter.reorderInlineTables": true,
|
|
27
|
+
"evenBetterToml.formatter.reorderKeys": true,
|
|
28
|
+
"python.languageServer": "None",
|
|
29
|
+
"python.testing.pytestArgs": [
|
|
30
|
+
"src",
|
|
31
|
+
"tests",
|
|
32
|
+
],
|
|
33
|
+
"python.testing.pytestEnabled": true,
|
|
34
|
+
"python.testing.unittestEnabled": false,
|
|
35
|
+
"ruff.importStrategy": "fromEnvironment"
|
|
36
|
+
}
|
pyflowx-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyflowx
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Lightweight, type-safe DAG task scheduler with multi-strategy execution.
|
|
5
|
+
Author: pyflowx
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: async,dag,scheduler,task,workflow
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Requires-Dist: graphlib-backport>=1.0.0; python_version < '3.9'
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: hatch>=1.14.2; extra == 'dev'
|
|
20
|
+
Requires-Dist: httpx>=0.28.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
22
|
+
Requires-Dist: prek>=0.4.5; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest-html>=4.1.1; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest-mock>=3.14.0; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-xdist>=3.6.1; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: ruff>=0.8.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: tox-uv>=1.13.1; extra == 'dev'
|
|
31
|
+
Requires-Dist: tox>=4.25.0; extra == 'dev'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# PyFlowX
|
|
35
|
+
|
|
36
|
+
> 轻量、类型安全的 DAG 任务调度器。
|
|
37
|
+
|
|
38
|
+
[](https://github.com/pyflowx/pyflowx/actions/workflows/ci.yml)
|
|
39
|
+
[](https://pypi.org/project/pyflowx/)
|
|
40
|
+
[](https://pypi.org/project/pyflowx/)
|
|
41
|
+
[](https://github.com/pyflowx/pyflowx)
|
|
42
|
+
[](https://github.com/pyflowx/pyflowx/blob/main/LICENSE)
|
|
43
|
+
|
|
44
|
+
PyFlowX 把"任务依赖"这件事做到极致简单:**参数名就是依赖声明**。无需装饰器、
|
|
45
|
+
无需样板包装器,写一个普通函数,框架按参数名自动注入上游结果。
|
|
46
|
+
|
|
47
|
+
## 特性
|
|
48
|
+
|
|
49
|
+
- **零样板** —— 参数名即依赖,框架自动注入上游结果
|
|
50
|
+
- **三种执行策略** —— `sequential`(调试)/ `thread`(I/O 密集同步)/ `async`(I/O 密集异步)
|
|
51
|
+
- **类型安全** —— `TaskSpec[T]` 把返回类型一路传到 `RunReport`,mypy strict 通过
|
|
52
|
+
- **DAG 校验** —— 构建时即时校验重名、缺失依赖、环
|
|
53
|
+
- **自动分层** —— Kahn 算法分组,同层任务可并行
|
|
54
|
+
- **重试与超时** —— 每个任务独立配置 `retries` 与 `timeout`
|
|
55
|
+
- **断点续跑** —— `MemoryBackend` / `JSONBackend`,成功结果可缓存复用
|
|
56
|
+
- **可观测** —— `on_event` 回调、`dry_run` 预览、Mermaid 可视化
|
|
57
|
+
- **零运行时依赖** —— 仅依赖标准库(3.8 需 `graphlib_backport`)
|
|
58
|
+
- **100% 测试覆盖** —— 分支覆盖率达 100%
|
|
59
|
+
|
|
60
|
+
## 安装
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install pyflowx
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
或使用 [uv](https://docs.astral.sh/uv/):
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
uv add pyflowx
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## 快速上手
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import pyflowx as px
|
|
76
|
+
|
|
77
|
+
def extract() -> list[int]:
|
|
78
|
+
return [1, 2, 3]
|
|
79
|
+
|
|
80
|
+
# 参数名 extract 自动匹配上游任务名 → 自动注入
|
|
81
|
+
def double(extract: list[int]) -> list[int]:
|
|
82
|
+
return [x * 2 for x in extract]
|
|
83
|
+
|
|
84
|
+
graph = px.Graph.from_specs([
|
|
85
|
+
px.TaskSpec("extract", extract),
|
|
86
|
+
px.TaskSpec("double", double, ("extract",)),
|
|
87
|
+
])
|
|
88
|
+
|
|
89
|
+
report = px.run(graph, strategy="sequential")
|
|
90
|
+
print(report["double"]) # [2, 4, 6]
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## 核心概念
|
|
94
|
+
|
|
95
|
+
### TaskSpec —— 任务描述
|
|
96
|
+
|
|
97
|
+
`TaskSpec` 是不可变的任务描述符,是唯一需要配置的东西:
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
px.TaskSpec(
|
|
101
|
+
name="fetch_user", # 唯一标识
|
|
102
|
+
fn=fetch_user, # 同步或异步函数
|
|
103
|
+
depends_on=("auth",), # 依赖的任务名
|
|
104
|
+
args=(uid,), # 静态位置参数(追加在注入参数后)
|
|
105
|
+
kwargs={"timeout": 30}, # 静态关键字参数
|
|
106
|
+
retries=3, # 失败重试次数(0 = 仅一次)
|
|
107
|
+
timeout=30.0, # 超时秒数(None = 不限制)
|
|
108
|
+
tags=("api", "user"), # 自由标签,用于子图过滤
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Graph —— DAG 构建
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
graph = px.Graph.from_specs([...]) # 整批校验(推荐)
|
|
116
|
+
# 或增量构建
|
|
117
|
+
graph = px.Graph()
|
|
118
|
+
graph.add(px.TaskSpec("a", fn_a))
|
|
119
|
+
graph.add(px.TaskSpec("b", fn_b, ("a",)))
|
|
120
|
+
|
|
121
|
+
graph.validate() # 显式校验(环检测)
|
|
122
|
+
graph.layers() # 拓扑分层
|
|
123
|
+
graph.to_mermaid() # Mermaid 可视化
|
|
124
|
+
graph.describe() # 人类可读摘要
|
|
125
|
+
graph.subgraph(("api",)) # 按标签切片
|
|
126
|
+
graph.subgraph_by_names(("a", "b")) # 按名称切片
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### run —— 执行
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
report = px.run(
|
|
133
|
+
graph,
|
|
134
|
+
strategy="async", # sequential | thread | async
|
|
135
|
+
max_workers=8, # thread 策略的线程池大小
|
|
136
|
+
dry_run=False, # True = 仅打印计划
|
|
137
|
+
on_event=callback, # 状态转换回调
|
|
138
|
+
state=px.JSONBackend("state.json"), # 断点续跑后端
|
|
139
|
+
)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### RunReport —— 结果
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
report["task_name"] # 任务返回值
|
|
146
|
+
report.result_of("task_name") # 完整 TaskResult
|
|
147
|
+
report.success # 整体是否成功
|
|
148
|
+
report.summary() # 统计字典
|
|
149
|
+
report.failed_tasks() # 失败任务名列表
|
|
150
|
+
report.describe() # 人类可读报告
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## 上下文注入规则
|
|
154
|
+
|
|
155
|
+
按顺序求值:
|
|
156
|
+
|
|
157
|
+
1. **标注为 `Context`** 的参数 → 接收完整上游结果映射
|
|
158
|
+
2. **名称匹配依赖** 的参数 → 接收该依赖的结果
|
|
159
|
+
3. **`**kwargs`** 参数 → 接收所有依赖结果(dict)
|
|
160
|
+
4. **`TaskSpec.args` / `kwargs`** → 为非依赖参数提供静态值
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from typing import Any, Dict
|
|
164
|
+
|
|
165
|
+
def aggregate(ctx: px.Context) -> Dict[str, Any]:
|
|
166
|
+
"""ctx 包含所有 depends_on 任务的返回值。"""
|
|
167
|
+
return dict(ctx)
|
|
168
|
+
|
|
169
|
+
def merge(fetch_a: str, fetch_b: str) -> str:
|
|
170
|
+
"""fetch_a / fetch_b 自动注入。"""
|
|
171
|
+
return fetch_a + fetch_b
|
|
172
|
+
|
|
173
|
+
def fetch_user(uid: int) -> dict: # uid 来自 TaskSpec.args
|
|
174
|
+
...
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## 执行策略对比
|
|
178
|
+
|
|
179
|
+
| 策略 | 并发模型 | 适用场景 | 同步任务 | 异步任务 |
|
|
180
|
+
|------|---------|---------|---------|---------|
|
|
181
|
+
| `sequential` | 串行 | 调试、CPU 密集 | 直接调用 | 事件循环 |
|
|
182
|
+
| `thread` | 线程池 | I/O 密集同步 | 线程池 | 不支持 |
|
|
183
|
+
| `async` | 事件循环 | I/O 密集异步 | 卸载到线程池 | 事件循环 |
|
|
184
|
+
|
|
185
|
+
所有策略都遵循 `retries`、`timeout`、上下文注入、状态后端,并发出 `TaskEvent`。
|
|
186
|
+
|
|
187
|
+
## 示例
|
|
188
|
+
|
|
189
|
+
仓库 `examples/` 目录包含完整示例:
|
|
190
|
+
|
|
191
|
+
- [`etl_pipeline.py`](examples/etl_pipeline.py) —— ETL 流水线(sequential)
|
|
192
|
+
- [`parallel_run.py`](examples/parallel_run.py) —— 并行执行对比(thread vs sequential)
|
|
193
|
+
- [`async_aggregation.py`](examples/async_aggregation.py) —— 异步聚合 + Context 注入
|
|
194
|
+
|
|
195
|
+
运行:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
python examples/etl_pipeline.py
|
|
199
|
+
python examples/parallel_run.py
|
|
200
|
+
python examples/async_aggregation.py
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## 断点续跑
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
from pyflowx import JSONBackend
|
|
207
|
+
|
|
208
|
+
# 第一次运行:成功结果写入 state.json
|
|
209
|
+
backend = JSONBackend("state.json")
|
|
210
|
+
report = px.run(graph, strategy="sequential", state=backend)
|
|
211
|
+
|
|
212
|
+
# 第二次运行:已缓存任务自动跳过
|
|
213
|
+
report = px.run(graph, strategy="sequential", state=backend)
|
|
214
|
+
# report.results 中缓存任务状态为 SKIPPED
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
## 错误处理
|
|
218
|
+
|
|
219
|
+
所有错误都是 `PyFlowXError` 的子类:
|
|
220
|
+
|
|
221
|
+
| 错误 | 触发时机 |
|
|
222
|
+
|------|---------|
|
|
223
|
+
| `DuplicateTaskError` | 任务名重复注册 |
|
|
224
|
+
| `MissingDependencyError` | 依赖了不存在的任务 |
|
|
225
|
+
| `CycleError` | 依赖图存在环 |
|
|
226
|
+
| `TaskFailedError` | 任务耗尽重试后仍失败 |
|
|
227
|
+
| `TaskTimeoutError` | 任务超时 |
|
|
228
|
+
| `InjectionError` | 上下文注入无法满足签名 |
|
|
229
|
+
| `StorageError` | 状态后端持久化失败 |
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
try:
|
|
233
|
+
report = px.run(graph, strategy="async")
|
|
234
|
+
except px.TaskFailedError as exc:
|
|
235
|
+
print(f"{exc.task} 失败: {exc.cause}(尝试 {exc.attempts} 次)")
|
|
236
|
+
except px.PyFlowXError:
|
|
237
|
+
# 捕获整个错误家族
|
|
238
|
+
raise
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
## 与其他库对比
|
|
242
|
+
|
|
243
|
+
| 特性 | PyFlowX | Airflow | Prefect | Dask |
|
|
244
|
+
|------|---------|---------|---------|------|
|
|
245
|
+
| 零样板 | 参数名即依赖 | 装饰器 + XCom | 装饰器 | 装饰器 |
|
|
246
|
+
| 运行时依赖 | 仅标准库 | 重型 | 中型 | 中型 |
|
|
247
|
+
| 类型安全 | mypy strict | 弱 | 中 | 中 |
|
|
248
|
+
| 异步原生 | 是 | 否 | 部分 | 否 |
|
|
249
|
+
| 断点续跑 | 内置 | 需配置 | 需配置 | 需配置 |
|
|
250
|
+
| 学习曲线 | 极低 | 高 | 中 | 中 |
|
|
251
|
+
| 适用规模 | 单机 | 集群 | 单机/集群 | 集群 |
|
|
252
|
+
|
|
253
|
+
PyFlowX 专注于**单机 DAG 调度**的极致简洁,适合 ETL、数据处理、CI 流水线等场景。
|
|
254
|
+
|
|
255
|
+
## 开发
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
# 安装开发依赖
|
|
259
|
+
uv sync --extra dev
|
|
260
|
+
|
|
261
|
+
# 运行测试(含覆盖率)
|
|
262
|
+
uv run pytest --cov=pyflowx --cov-fail-under=100
|
|
263
|
+
|
|
264
|
+
# 类型检查
|
|
265
|
+
uv run mypy
|
|
266
|
+
|
|
267
|
+
# 代码风格
|
|
268
|
+
uv run ruff check src tests examples
|
|
269
|
+
uv run ruff format --check src tests examples
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## 许可证
|
|
273
|
+
|
|
274
|
+
MIT
|