pipeline-frame 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipeline_frame-0.1.0/.github/workflows/publish.yml +101 -0
- pipeline_frame-0.1.0/.gitignore +29 -0
- pipeline_frame-0.1.0/.python-version +1 -0
- pipeline_frame-0.1.0/PKG-INFO +93 -0
- pipeline_frame-0.1.0/README.md +82 -0
- pipeline_frame-0.1.0/pipeline_frame/__init__.py +30 -0
- pipeline_frame-0.1.0/pipeline_frame/context.py +27 -0
- pipeline_frame-0.1.0/pipeline_frame/errors.py +24 -0
- pipeline_frame-0.1.0/pipeline_frame/models.py +91 -0
- pipeline_frame-0.1.0/pipeline_frame/pipeline.py +187 -0
- pipeline_frame-0.1.0/pipeline_frame/step.py +47 -0
- pipeline_frame-0.1.0/pyproject.toml +22 -0
- pipeline_frame-0.1.0/tests/test_pipeline.py +189 -0
- pipeline_frame-0.1.0/uv.lock +8 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
tags:
|
|
8
|
+
- "v*"
|
|
9
|
+
pull_request:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
name: Test on Python ${{ matrix.python-version }}
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
19
|
+
|
|
20
|
+
steps:
|
|
21
|
+
- name: Check out repository
|
|
22
|
+
uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up Python
|
|
25
|
+
uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Run tests
|
|
30
|
+
run: python -m unittest discover -s tests -v
|
|
31
|
+
|
|
32
|
+
build:
|
|
33
|
+
name: Build distributions
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
needs: test
|
|
36
|
+
|
|
37
|
+
steps:
|
|
38
|
+
- name: Check out repository
|
|
39
|
+
uses: actions/checkout@v4
|
|
40
|
+
|
|
41
|
+
- name: Set up Python
|
|
42
|
+
uses: actions/setup-python@v5
|
|
43
|
+
with:
|
|
44
|
+
python-version: "3.11"
|
|
45
|
+
|
|
46
|
+
- name: Validate tag version matches package version
|
|
47
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
48
|
+
run: |
|
|
49
|
+
python - <<'PY'
|
|
50
|
+
import pathlib
|
|
51
|
+
import re
|
|
52
|
+
|
|
53
|
+
pyproject = pathlib.Path("pyproject.toml").read_text(encoding="utf-8")
|
|
54
|
+
match = re.search(r'^version\s*=\s*"([^"]+)"', pyproject, re.MULTILINE)
|
|
55
|
+
if match is None:
|
|
56
|
+
raise SystemExit("Could not find project version in pyproject.toml")
|
|
57
|
+
|
|
58
|
+
package_version = match.group(1)
|
|
59
|
+
tag = "${{ github.ref_name }}"
|
|
60
|
+
tag_version = tag.removeprefix("v")
|
|
61
|
+
|
|
62
|
+
if tag_version != package_version:
|
|
63
|
+
raise SystemExit(
|
|
64
|
+
f"Tag version '{tag_version}' does not match package version '{package_version}'"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
print(f"Version check passed: tag={tag}, package={package_version}")
|
|
68
|
+
PY
|
|
69
|
+
|
|
70
|
+
- name: Install build tools
|
|
71
|
+
run: python -m pip install --upgrade build twine
|
|
72
|
+
|
|
73
|
+
- name: Build distributions
|
|
74
|
+
run: python -m build
|
|
75
|
+
|
|
76
|
+
- name: Check distributions
|
|
77
|
+
run: python -m twine check dist/*
|
|
78
|
+
|
|
79
|
+
- name: Upload build artifacts
|
|
80
|
+
uses: actions/upload-artifact@v4
|
|
81
|
+
with:
|
|
82
|
+
name: python-package-distributions
|
|
83
|
+
path: dist/
|
|
84
|
+
|
|
85
|
+
publish:
|
|
86
|
+
name: Publish to PyPI
|
|
87
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
88
|
+
runs-on: ubuntu-latest
|
|
89
|
+
needs: build
|
|
90
|
+
permissions:
|
|
91
|
+
id-token: write
|
|
92
|
+
|
|
93
|
+
steps:
|
|
94
|
+
- name: Download build artifacts
|
|
95
|
+
uses: actions/download-artifact@v4
|
|
96
|
+
with:
|
|
97
|
+
name: python-package-distributions
|
|
98
|
+
path: dist/
|
|
99
|
+
|
|
100
|
+
- name: Publish distributions to PyPI
|
|
101
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Python cache and compiled files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Build artifacts
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
env/
|
|
16
|
+
|
|
17
|
+
# Tool caches
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.ruff_cache/
|
|
21
|
+
.coverage
|
|
22
|
+
htmlcov/
|
|
23
|
+
|
|
24
|
+
# IDE files
|
|
25
|
+
.idea/
|
|
26
|
+
.vscode/
|
|
27
|
+
|
|
28
|
+
# OS files
|
|
29
|
+
.DS_Store
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pipeline-frame
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight in-memory pipeline framework built around before, run, and after.
|
|
5
|
+
Project-URL: Homepage, https://github.com/al6nlee/pipeline-frame
|
|
6
|
+
Project-URL: Repository, https://github.com/al6nlee/pipeline-frame
|
|
7
|
+
Author-email: alan <al6nlee@gmail.com>
|
|
8
|
+
Keywords: pipeline,pipeline-frame,workflow
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# pipeline-frame
|
|
13
|
+
|
|
14
|
+
`pipeline-frame` is a lightweight in-memory pipeline framework for Python 3.11+.
|
|
15
|
+
|
|
16
|
+
It keeps the model intentionally narrow: every step is built around `before()`, `run()`, and `after()`.
|
|
17
|
+
|
|
18
|
+
## Core idea
|
|
19
|
+
|
|
20
|
+
Define a linear pipeline with `Step` instances and `|` composition:
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from pipeline_frame import Pipeline, PipelineContext, Step
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class A(Step):
|
|
27
|
+
def before(self, context: PipelineContext) -> None:
|
|
28
|
+
print("a.before")
|
|
29
|
+
|
|
30
|
+
def run(self, context: PipelineContext) -> None:
|
|
31
|
+
context.set("text", context.input["text"].strip())
|
|
32
|
+
|
|
33
|
+
def after(self, context: PipelineContext) -> None:
|
|
34
|
+
print("a.after")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class B(Step):
|
|
38
|
+
def run(self, context: PipelineContext) -> None:
|
|
39
|
+
context.set_output(context.get("text").upper())
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
pipeline = A() | B()
|
|
43
|
+
|
|
44
|
+
result = pipeline.run({"text": " hello world "})
|
|
45
|
+
print(result.output)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Execution is linear and easy to reason about:
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
A.before()
|
|
52
|
+
A.run()
|
|
53
|
+
A.after()
|
|
54
|
+
B.before()
|
|
55
|
+
B.run()
|
|
56
|
+
B.after()
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
If you want to keep an explicit pipeline name, you can also write:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
pipeline = Pipeline("request") | A() | B()
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Design goals
|
|
66
|
+
|
|
67
|
+
- one core abstraction: `Pipeline`
|
|
68
|
+
- one step lifecycle: `before`, `run`, `after`
|
|
69
|
+
- chainable definitions with minimal ceremony
|
|
70
|
+
- structured execution results
|
|
71
|
+
- predictable stop-on-failure behavior
|
|
72
|
+
|
|
73
|
+
## Failure behavior
|
|
74
|
+
|
|
75
|
+
- if `before()` fails, the current step stops and later steps do not run
|
|
76
|
+
- if `run()` fails, the current step still attempts `after()`
|
|
77
|
+
- if `after()` fails, the pipeline is marked failed and later steps do not run
|
|
78
|
+
- all details are available on `PipelineResult`
|
|
79
|
+
|
|
80
|
+
## API overview
|
|
81
|
+
|
|
82
|
+
- `Pipeline`: ordered collection of step instances
|
|
83
|
+
- `Step`: base class with `before()`, `run()`, and `after()`
|
|
84
|
+
- `PipelineContext`: shared input, state, output, and error
|
|
85
|
+
- `PipelineResult`: execution report for one run
|
|
86
|
+
|
|
87
|
+
## Development
|
|
88
|
+
|
|
89
|
+
Run tests from the repository root:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
python -m unittest discover -s tests -v
|
|
93
|
+
```
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# pipeline-frame
|
|
2
|
+
|
|
3
|
+
`pipeline-frame` is a lightweight in-memory pipeline framework for Python 3.11+.
|
|
4
|
+
|
|
5
|
+
It keeps the model intentionally narrow: every step is built around `before()`, `run()`, and `after()`.
|
|
6
|
+
|
|
7
|
+
## Core idea
|
|
8
|
+
|
|
9
|
+
Define a linear pipeline with `Step` instances and `|` composition:
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from pipeline_frame import Pipeline, PipelineContext, Step
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class A(Step):
|
|
16
|
+
def before(self, context: PipelineContext) -> None:
|
|
17
|
+
print("a.before")
|
|
18
|
+
|
|
19
|
+
def run(self, context: PipelineContext) -> None:
|
|
20
|
+
context.set("text", context.input["text"].strip())
|
|
21
|
+
|
|
22
|
+
def after(self, context: PipelineContext) -> None:
|
|
23
|
+
print("a.after")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class B(Step):
|
|
27
|
+
def run(self, context: PipelineContext) -> None:
|
|
28
|
+
context.set_output(context.get("text").upper())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
pipeline = A() | B()
|
|
32
|
+
|
|
33
|
+
result = pipeline.run({"text": " hello world "})
|
|
34
|
+
print(result.output)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Execution is linear and easy to reason about:
|
|
38
|
+
|
|
39
|
+
```text
|
|
40
|
+
A.before()
|
|
41
|
+
A.run()
|
|
42
|
+
A.after()
|
|
43
|
+
B.before()
|
|
44
|
+
B.run()
|
|
45
|
+
B.after()
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
If you want to keep an explicit pipeline name, you can also write:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
pipeline = Pipeline("request") | A() | B()
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Design goals
|
|
55
|
+
|
|
56
|
+
- one core abstraction: `Pipeline`
|
|
57
|
+
- one step lifecycle: `before`, `run`, `after`
|
|
58
|
+
- chainable definitions with minimal ceremony
|
|
59
|
+
- structured execution results
|
|
60
|
+
- predictable stop-on-failure behavior
|
|
61
|
+
|
|
62
|
+
## Failure behavior
|
|
63
|
+
|
|
64
|
+
- if `before()` fails, the current step stops and later steps do not run
|
|
65
|
+
- if `run()` fails, the current step still attempts `after()`
|
|
66
|
+
- if `after()` fails, the pipeline is marked failed and later steps do not run
|
|
67
|
+
- all details are available on `PipelineResult`
|
|
68
|
+
|
|
69
|
+
## API overview
|
|
70
|
+
|
|
71
|
+
- `Pipeline`: ordered collection of step instances
|
|
72
|
+
- `Step`: base class with `before()`, `run()`, and `after()`
|
|
73
|
+
- `PipelineContext`: shared input, state, output, and error
|
|
74
|
+
- `PipelineResult`: execution report for one run
|
|
75
|
+
|
|
76
|
+
## Development
|
|
77
|
+
|
|
78
|
+
Run tests from the repository root:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python -m unittest discover -s tests -v
|
|
82
|
+
```
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from .context import PipelineContext, RunContext
|
|
2
|
+
from .errors import PipelineDefinitionError, PipelineFailedError
|
|
3
|
+
from .models import (
|
|
4
|
+
PhaseName,
|
|
5
|
+
PhaseResult,
|
|
6
|
+
PhaseStatus,
|
|
7
|
+
PipelineResult,
|
|
8
|
+
RunStatus,
|
|
9
|
+
StepResult,
|
|
10
|
+
StepStatus,
|
|
11
|
+
)
|
|
12
|
+
from .pipeline import Pipeline
|
|
13
|
+
from .step import Step, StepSpec
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"PhaseName",
|
|
17
|
+
"PhaseResult",
|
|
18
|
+
"PhaseStatus",
|
|
19
|
+
"Pipeline",
|
|
20
|
+
"PipelineContext",
|
|
21
|
+
"PipelineDefinitionError",
|
|
22
|
+
"PipelineFailedError",
|
|
23
|
+
"PipelineResult",
|
|
24
|
+
"RunContext",
|
|
25
|
+
"RunStatus",
|
|
26
|
+
"Step",
|
|
27
|
+
"StepSpec",
|
|
28
|
+
"StepResult",
|
|
29
|
+
"StepStatus",
|
|
30
|
+
]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass(slots=True)
|
|
6
|
+
class PipelineContext:
|
|
7
|
+
"""Mutable state shared by one pipeline run."""
|
|
8
|
+
|
|
9
|
+
input: Any = None
|
|
10
|
+
state: dict[str, Any] = field(default_factory=dict)
|
|
11
|
+
output: Any = None
|
|
12
|
+
error: BaseException | None = None
|
|
13
|
+
|
|
14
|
+
def set(self, key: str, value: Any) -> None:
|
|
15
|
+
self.state[key] = value
|
|
16
|
+
|
|
17
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
18
|
+
return self.state.get(key, default)
|
|
19
|
+
|
|
20
|
+
def set_output(self, value: Any) -> None:
|
|
21
|
+
self.output = value
|
|
22
|
+
|
|
23
|
+
def fail(self, error: BaseException) -> None:
|
|
24
|
+
self.error = error
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
RunContext = PipelineContext
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
if TYPE_CHECKING:
|
|
4
|
+
from .models import PipelineResult
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PipelineDefinitionError(ValueError):
|
|
8
|
+
"""Raised when a pipeline definition is invalid."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PipelineFailedError(RuntimeError):
|
|
12
|
+
"""Raised when a pipeline run fails."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, result: "PipelineResult", cause: BaseException):
|
|
15
|
+
message = f"Pipeline '{result.pipeline_name}' failed"
|
|
16
|
+
if result.failure_step and result.failure_phase:
|
|
17
|
+
message += (
|
|
18
|
+
f" during {result.failure_phase.value.lower()} "
|
|
19
|
+
f"of '{result.failure_step}'"
|
|
20
|
+
)
|
|
21
|
+
message += f": {cause}"
|
|
22
|
+
super().__init__(message)
|
|
23
|
+
self.result = result
|
|
24
|
+
self.cause = cause
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from enum import StrEnum
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class PhaseName(StrEnum):
|
|
7
|
+
BEFORE = "BEFORE"
|
|
8
|
+
RUN = "RUN"
|
|
9
|
+
AFTER = "AFTER"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PhaseStatus(StrEnum):
|
|
13
|
+
PENDING = "PENDING"
|
|
14
|
+
SKIPPED = "SKIPPED"
|
|
15
|
+
RUNNING = "RUNNING"
|
|
16
|
+
SUCCESS = "SUCCESS"
|
|
17
|
+
FAILED = "FAILED"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class RunStatus(StrEnum):
|
|
21
|
+
SUCCESS = "SUCCESS"
|
|
22
|
+
FAILED = "FAILED"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(slots=True)
|
|
26
|
+
class PhaseResult:
|
|
27
|
+
status: PhaseStatus = PhaseStatus.PENDING
|
|
28
|
+
error: BaseException | None = None
|
|
29
|
+
started_at: float | None = None
|
|
30
|
+
finished_at: float | None = None
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def duration(self) -> float | None:
|
|
34
|
+
if self.started_at is None or self.finished_at is None:
|
|
35
|
+
return None
|
|
36
|
+
return self.finished_at - self.started_at
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(slots=True)
|
|
40
|
+
class StepResult:
|
|
41
|
+
name: str
|
|
42
|
+
before: PhaseResult = field(default_factory=PhaseResult)
|
|
43
|
+
run: PhaseResult = field(default_factory=PhaseResult)
|
|
44
|
+
after: PhaseResult = field(default_factory=PhaseResult)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def succeeded(self) -> bool:
|
|
48
|
+
return self.run.status == PhaseStatus.SUCCESS and self.after.status in {
|
|
49
|
+
PhaseStatus.SUCCESS,
|
|
50
|
+
PhaseStatus.SKIPPED,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def failed(self) -> bool:
|
|
55
|
+
return any(
|
|
56
|
+
phase.status == PhaseStatus.FAILED
|
|
57
|
+
for phase in (self.before, self.run, self.after)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(slots=True)
|
|
62
|
+
class PipelineResult:
|
|
63
|
+
pipeline_name: str
|
|
64
|
+
status: RunStatus
|
|
65
|
+
input_value: Any
|
|
66
|
+
output: Any
|
|
67
|
+
state: dict[str, Any]
|
|
68
|
+
error: BaseException | None
|
|
69
|
+
step_results: dict[str, StepResult]
|
|
70
|
+
executed_steps: tuple[str, ...]
|
|
71
|
+
started_at: float
|
|
72
|
+
finished_at: float
|
|
73
|
+
failure_step: str | None = None
|
|
74
|
+
failure_phase: PhaseName | None = None
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def duration(self) -> float:
|
|
78
|
+
return self.finished_at - self.started_at
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def input(self) -> Any:
|
|
82
|
+
return self.input_value
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def failed_step_result(self) -> StepResult | None:
|
|
86
|
+
if self.failure_step is None:
|
|
87
|
+
return None
|
|
88
|
+
return self.step_results[self.failure_step]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
StepStatus = PhaseStatus
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import Any, Self
|
|
5
|
+
|
|
6
|
+
from .context import PipelineContext
|
|
7
|
+
from .errors import PipelineDefinitionError, PipelineFailedError
|
|
8
|
+
from .models import PhaseName, PhaseResult, PhaseStatus, PipelineResult, RunStatus, StepResult
|
|
9
|
+
from .step import Step, StepSpec
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Pipeline:
|
|
13
|
+
"""A small linear pipeline centered on before, run, and after."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, name: str = "pipeline"):
|
|
16
|
+
self.name = name
|
|
17
|
+
self._steps: list[StepSpec | Step] = []
|
|
18
|
+
|
|
19
|
+
def add(
|
|
20
|
+
self,
|
|
21
|
+
step: type[Step] | Step,
|
|
22
|
+
/,
|
|
23
|
+
*,
|
|
24
|
+
name: str | None = None,
|
|
25
|
+
**options: Any,
|
|
26
|
+
) -> Self:
|
|
27
|
+
if isinstance(step, Step):
|
|
28
|
+
if name is not None or options:
|
|
29
|
+
raise TypeError("Cannot override name or options when adding a Step instance")
|
|
30
|
+
|
|
31
|
+
step_name = step.name or step.__class__.name or step.__class__.__name__
|
|
32
|
+
if any(existing.name == step_name for existing in self._steps):
|
|
33
|
+
raise PipelineDefinitionError(f"Duplicate step name: '{step_name}'")
|
|
34
|
+
|
|
35
|
+
step.name = step_name
|
|
36
|
+
self._steps.append(step)
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
if not issubclass(step, Step):
|
|
40
|
+
raise TypeError("step must be a Step subclass or Step instance")
|
|
41
|
+
|
|
42
|
+
step_name = name or step.name or step.__name__
|
|
43
|
+
if any(step.name == step_name for step in self._steps):
|
|
44
|
+
raise PipelineDefinitionError(f"Duplicate step name: '{step_name}'")
|
|
45
|
+
|
|
46
|
+
self._steps.append(
|
|
47
|
+
StepSpec(
|
|
48
|
+
step_type=step,
|
|
49
|
+
name=step_name,
|
|
50
|
+
options=dict(options),
|
|
51
|
+
)
|
|
52
|
+
)
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def add_many(self, *steps: type[Step] | Step) -> Self:
|
|
56
|
+
for step in steps:
|
|
57
|
+
self.add(step)
|
|
58
|
+
return self
|
|
59
|
+
|
|
60
|
+
pipe = add
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def steps(self) -> tuple[StepSpec | Step, ...]:
|
|
64
|
+
return tuple(self._steps)
|
|
65
|
+
|
|
66
|
+
def __or__(self, other):
|
|
67
|
+
cloned = self._clone()
|
|
68
|
+
cloned.add(other)
|
|
69
|
+
return cloned
|
|
70
|
+
|
|
71
|
+
def run(self, input_value=None, /, *, raise_on_error: bool = True) -> PipelineResult:
|
|
72
|
+
if not self._steps:
|
|
73
|
+
raise PipelineDefinitionError("Pipeline must contain at least one step")
|
|
74
|
+
|
|
75
|
+
context = PipelineContext(input=input_value)
|
|
76
|
+
step_instances = [self._materialize(step) for step in self._steps]
|
|
77
|
+
step_results = {step.name: StepResult(name=step.name) for step in step_instances}
|
|
78
|
+
executed_steps: list[str] = []
|
|
79
|
+
started_at = time.perf_counter()
|
|
80
|
+
first_error: BaseException | None = None
|
|
81
|
+
failure_step: str | None = None
|
|
82
|
+
failure_phase: PhaseName | None = None
|
|
83
|
+
|
|
84
|
+
for step in step_instances:
|
|
85
|
+
step_result = step_results[step.name]
|
|
86
|
+
|
|
87
|
+
before_error = self._run_callable(step.before, context, step_result.before)
|
|
88
|
+
if before_error is not None:
|
|
89
|
+
first_error = before_error
|
|
90
|
+
failure_step = step.name
|
|
91
|
+
failure_phase = PhaseName.BEFORE
|
|
92
|
+
context.fail(first_error)
|
|
93
|
+
break
|
|
94
|
+
|
|
95
|
+
run_error = self._run_callable(step.run, context, step_result.run)
|
|
96
|
+
after_error = self._run_callable(step.after, context, step_result.after)
|
|
97
|
+
executed_steps.append(step.name)
|
|
98
|
+
|
|
99
|
+
cause = self._combine_errors(step.name, run_error, after_error)
|
|
100
|
+
if cause is None:
|
|
101
|
+
continue
|
|
102
|
+
|
|
103
|
+
first_error = cause
|
|
104
|
+
failure_step = step.name
|
|
105
|
+
failure_phase = PhaseName.RUN if run_error is not None else PhaseName.AFTER
|
|
106
|
+
context.fail(first_error)
|
|
107
|
+
break
|
|
108
|
+
|
|
109
|
+
self._finalize_pending_results(step_results)
|
|
110
|
+
finished_at = time.perf_counter()
|
|
111
|
+
status = RunStatus.FAILED if first_error is not None else RunStatus.SUCCESS
|
|
112
|
+
result = PipelineResult(
|
|
113
|
+
pipeline_name=self.name,
|
|
114
|
+
status=status,
|
|
115
|
+
input_value=context.input,
|
|
116
|
+
output=context.output,
|
|
117
|
+
state=dict(context.state),
|
|
118
|
+
error=first_error,
|
|
119
|
+
step_results=step_results,
|
|
120
|
+
executed_steps=tuple(executed_steps),
|
|
121
|
+
started_at=started_at,
|
|
122
|
+
finished_at=finished_at,
|
|
123
|
+
failure_step=failure_step,
|
|
124
|
+
failure_phase=failure_phase,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
if first_error is not None and raise_on_error:
|
|
128
|
+
raise PipelineFailedError(result=result, cause=first_error)
|
|
129
|
+
|
|
130
|
+
return result
|
|
131
|
+
|
|
132
|
+
@staticmethod
|
|
133
|
+
def _run_callable(
|
|
134
|
+
action,
|
|
135
|
+
context: PipelineContext,
|
|
136
|
+
result: PhaseResult,
|
|
137
|
+
) -> BaseException | None:
|
|
138
|
+
result.status = PhaseStatus.RUNNING
|
|
139
|
+
result.started_at = time.perf_counter()
|
|
140
|
+
try:
|
|
141
|
+
action(context)
|
|
142
|
+
except Exception as error:
|
|
143
|
+
result.status = PhaseStatus.FAILED
|
|
144
|
+
result.error = error
|
|
145
|
+
result.finished_at = time.perf_counter()
|
|
146
|
+
return error
|
|
147
|
+
|
|
148
|
+
result.status = PhaseStatus.SUCCESS
|
|
149
|
+
result.error = None
|
|
150
|
+
result.finished_at = time.perf_counter()
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def _combine_errors(
|
|
155
|
+
step_name: str,
|
|
156
|
+
run_error: BaseException | None,
|
|
157
|
+
after_error: BaseException | None,
|
|
158
|
+
) -> BaseException | None:
|
|
159
|
+
if run_error is None:
|
|
160
|
+
return after_error
|
|
161
|
+
if after_error is None:
|
|
162
|
+
return run_error
|
|
163
|
+
return ExceptionGroup(
|
|
164
|
+
f"Step '{step_name}' failed in run and after",
|
|
165
|
+
[run_error, after_error],
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
def _finalize_pending_results(step_results: dict[str, StepResult]) -> None:
|
|
170
|
+
for result in step_results.values():
|
|
171
|
+
if result.before.status == PhaseStatus.PENDING:
|
|
172
|
+
result.before.status = PhaseStatus.SKIPPED
|
|
173
|
+
if result.run.status == PhaseStatus.PENDING:
|
|
174
|
+
result.run.status = PhaseStatus.SKIPPED
|
|
175
|
+
if result.after.status == PhaseStatus.PENDING:
|
|
176
|
+
result.after.status = PhaseStatus.SKIPPED
|
|
177
|
+
|
|
178
|
+
def _clone(self) -> "Pipeline":
|
|
179
|
+
cloned = Pipeline(self.name)
|
|
180
|
+
cloned._steps = list(self._steps)
|
|
181
|
+
return cloned
|
|
182
|
+
|
|
183
|
+
@staticmethod
|
|
184
|
+
def _materialize(step: StepSpec | Step) -> Step:
|
|
185
|
+
if isinstance(step, StepSpec):
|
|
186
|
+
return step.create()
|
|
187
|
+
return step
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .context import PipelineContext
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Step:
|
|
8
|
+
"""Base class for one linear pipeline step."""
|
|
9
|
+
|
|
10
|
+
name: str | None = None
|
|
11
|
+
|
|
12
|
+
def __init_subclass__(cls, **kwargs):
|
|
13
|
+
super().__init_subclass__(**kwargs)
|
|
14
|
+
if cls is not Step and cls.name is None:
|
|
15
|
+
cls.name = cls.__name__
|
|
16
|
+
|
|
17
|
+
def before(self, context: PipelineContext) -> None:
|
|
18
|
+
"""Run before the main step logic."""
|
|
19
|
+
|
|
20
|
+
def run(self, context: PipelineContext) -> None:
|
|
21
|
+
"""Run the main step logic."""
|
|
22
|
+
|
|
23
|
+
def after(self, context: PipelineContext) -> None:
|
|
24
|
+
"""Run after the main step logic."""
|
|
25
|
+
|
|
26
|
+
def __or__(self, other):
|
|
27
|
+
from .pipeline import Pipeline
|
|
28
|
+
|
|
29
|
+
return Pipeline() | self | other
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class StepSpec:
|
|
34
|
+
step_type: type[Step]
|
|
35
|
+
name: str
|
|
36
|
+
options: dict[str, Any] = field(default_factory=dict)
|
|
37
|
+
|
|
38
|
+
def __post_init__(self) -> None:
|
|
39
|
+
if not issubclass(self.step_type, Step):
|
|
40
|
+
raise TypeError("step_type must be a Step subclass")
|
|
41
|
+
if not self.name:
|
|
42
|
+
raise ValueError("Step name cannot be empty")
|
|
43
|
+
|
|
44
|
+
def create(self) -> Step:
|
|
45
|
+
step = self.step_type(**self.options)
|
|
46
|
+
step.name = self.name
|
|
47
|
+
return step
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pipeline-frame"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A lightweight in-memory pipeline framework built around before, run, and after."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "alan", email = "al6nlee@gmail.com" }
|
|
9
|
+
]
|
|
10
|
+
keywords = ["pipeline-frame", "pipeline", "workflow"]
|
|
11
|
+
dependencies = []
|
|
12
|
+
|
|
13
|
+
[project.urls]
|
|
14
|
+
Homepage = "https://github.com/al6nlee/pipeline-frame"
|
|
15
|
+
Repository = "https://github.com/al6nlee/pipeline-frame"
|
|
16
|
+
|
|
17
|
+
[build-system]
|
|
18
|
+
requires = ["hatchling"]
|
|
19
|
+
build-backend = "hatchling.build"
|
|
20
|
+
|
|
21
|
+
[tool.hatch.build.targets.wheel]
|
|
22
|
+
packages = ["pipeline_frame"]
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from pipeline_frame import (
|
|
4
|
+
PhaseName,
|
|
5
|
+
PhaseStatus,
|
|
6
|
+
Pipeline,
|
|
7
|
+
PipelineContext,
|
|
8
|
+
PipelineDefinitionError,
|
|
9
|
+
PipelineFailedError,
|
|
10
|
+
RunStatus,
|
|
11
|
+
Step,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def append_trace(context: PipelineContext, item: str) -> None:
|
|
16
|
+
trace = context.get("trace")
|
|
17
|
+
if trace is None:
|
|
18
|
+
trace = []
|
|
19
|
+
context.set("trace", trace)
|
|
20
|
+
trace.append(item)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class A(Step):
|
|
24
|
+
def before(self, context: PipelineContext) -> None:
|
|
25
|
+
append_trace(context, "a.before")
|
|
26
|
+
|
|
27
|
+
def run(self, context: PipelineContext) -> None:
|
|
28
|
+
append_trace(context, "a.run")
|
|
29
|
+
context.set("text", context.input["text"].strip())
|
|
30
|
+
|
|
31
|
+
def after(self, context: PipelineContext) -> None:
|
|
32
|
+
append_trace(context, "a.after")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class B(Step):
|
|
36
|
+
def before(self, context: PipelineContext) -> None:
|
|
37
|
+
append_trace(context, "b.before")
|
|
38
|
+
|
|
39
|
+
def run(self, context: PipelineContext) -> None:
|
|
40
|
+
append_trace(context, "b.run")
|
|
41
|
+
context.set("text", context.get("text").upper())
|
|
42
|
+
|
|
43
|
+
def after(self, context: PipelineContext) -> None:
|
|
44
|
+
append_trace(context, "b.after")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class C(Step):
|
|
48
|
+
def before(self, context: PipelineContext) -> None:
|
|
49
|
+
append_trace(context, "c.before")
|
|
50
|
+
|
|
51
|
+
def run(self, context: PipelineContext) -> None:
|
|
52
|
+
append_trace(context, "c.run")
|
|
53
|
+
context.set_output(f"RESULT={context.get('text')}")
|
|
54
|
+
|
|
55
|
+
def after(self, context: PipelineContext) -> None:
|
|
56
|
+
append_trace(context, "c.after")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class ExplodeOnRun(Step):
|
|
60
|
+
def before(self, context: PipelineContext) -> None:
|
|
61
|
+
append_trace(context, "explode.before")
|
|
62
|
+
|
|
63
|
+
def run(self, context: PipelineContext) -> None:
|
|
64
|
+
append_trace(context, "explode.run")
|
|
65
|
+
raise ValueError("boom")
|
|
66
|
+
|
|
67
|
+
def after(self, context: PipelineContext) -> None:
|
|
68
|
+
append_trace(context, "explode.after")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class BrokenAfter(Step):
|
|
72
|
+
def run(self, context: PipelineContext) -> None:
|
|
73
|
+
append_trace(context, "broken.run")
|
|
74
|
+
|
|
75
|
+
def after(self, context: PipelineContext) -> None:
|
|
76
|
+
append_trace(context, "broken.after")
|
|
77
|
+
raise RuntimeError("after failed")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ExplodeOnBefore(Step):
|
|
81
|
+
def before(self, context: PipelineContext) -> None:
|
|
82
|
+
append_trace(context, "before.fail")
|
|
83
|
+
raise RuntimeError("before failed")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class PipelineTests(unittest.TestCase):
|
|
87
|
+
def test_pipeline_runs_step_lifecycle_in_order(self) -> None:
|
|
88
|
+
pipeline = Pipeline("request").add(A).add(B).add(C)
|
|
89
|
+
|
|
90
|
+
result = pipeline.run({"text": " hello "})
|
|
91
|
+
|
|
92
|
+
self.assertEqual(result.status, RunStatus.SUCCESS)
|
|
93
|
+
self.assertEqual(result.output, "RESULT=HELLO")
|
|
94
|
+
self.assertEqual(
|
|
95
|
+
result.state["trace"],
|
|
96
|
+
[
|
|
97
|
+
"a.before",
|
|
98
|
+
"a.run",
|
|
99
|
+
"a.after",
|
|
100
|
+
"b.before",
|
|
101
|
+
"b.run",
|
|
102
|
+
"b.after",
|
|
103
|
+
"c.before",
|
|
104
|
+
"c.run",
|
|
105
|
+
"c.after",
|
|
106
|
+
],
|
|
107
|
+
)
|
|
108
|
+
self.assertEqual(result.executed_steps, ("A", "B", "C"))
|
|
109
|
+
|
|
110
|
+
def test_run_failure_stops_later_steps_but_keeps_current_after(self) -> None:
|
|
111
|
+
pipeline = Pipeline("request").add(ExplodeOnRun).add(C)
|
|
112
|
+
|
|
113
|
+
with self.assertRaises(PipelineFailedError) as captured:
|
|
114
|
+
pipeline.run()
|
|
115
|
+
|
|
116
|
+
result = captured.exception.result
|
|
117
|
+
self.assertEqual(result.status, RunStatus.FAILED)
|
|
118
|
+
self.assertEqual(result.failure_step, "ExplodeOnRun")
|
|
119
|
+
self.assertEqual(result.failure_phase, PhaseName.RUN)
|
|
120
|
+
self.assertEqual(result.executed_steps, ("ExplodeOnRun",))
|
|
121
|
+
self.assertEqual(
|
|
122
|
+
result.state["trace"],
|
|
123
|
+
["explode.before", "explode.run", "explode.after"],
|
|
124
|
+
)
|
|
125
|
+
self.assertEqual(result.step_results["C"].before.status, PhaseStatus.SKIPPED)
|
|
126
|
+
|
|
127
|
+
def test_after_failure_marks_run_failed(self) -> None:
|
|
128
|
+
pipeline = Pipeline("request").add(BrokenAfter)
|
|
129
|
+
|
|
130
|
+
with self.assertRaises(PipelineFailedError) as captured:
|
|
131
|
+
pipeline.run()
|
|
132
|
+
|
|
133
|
+
result = captured.exception.result
|
|
134
|
+
self.assertEqual(result.failure_step, "BrokenAfter")
|
|
135
|
+
self.assertEqual(result.failure_phase, PhaseName.AFTER)
|
|
136
|
+
self.assertEqual(
|
|
137
|
+
result.state["trace"],
|
|
138
|
+
["broken.run", "broken.after"],
|
|
139
|
+
)
|
|
140
|
+
self.assertEqual(result.step_results["BrokenAfter"].after.status, PhaseStatus.FAILED)
|
|
141
|
+
|
|
142
|
+
def test_before_failure_stops_current_step_immediately(self) -> None:
|
|
143
|
+
pipeline = Pipeline("request").add(ExplodeOnBefore).add(C)
|
|
144
|
+
|
|
145
|
+
with self.assertRaises(PipelineFailedError) as captured:
|
|
146
|
+
pipeline.run()
|
|
147
|
+
|
|
148
|
+
result = captured.exception.result
|
|
149
|
+
self.assertEqual(result.failure_step, "ExplodeOnBefore")
|
|
150
|
+
self.assertEqual(result.failure_phase, PhaseName.BEFORE)
|
|
151
|
+
self.assertEqual(result.state["trace"], ["before.fail"])
|
|
152
|
+
self.assertEqual(result.step_results["ExplodeOnBefore"].run.status, PhaseStatus.SKIPPED)
|
|
153
|
+
self.assertEqual(result.step_results["ExplodeOnBefore"].after.status, PhaseStatus.SKIPPED)
|
|
154
|
+
|
|
155
|
+
def test_duplicate_step_names_are_rejected(self) -> None:
|
|
156
|
+
pipeline = Pipeline("request").add(A)
|
|
157
|
+
|
|
158
|
+
with self.assertRaises(PipelineDefinitionError):
|
|
159
|
+
pipeline.add(A)
|
|
160
|
+
|
|
161
|
+
def test_pipe_operator_builds_pipeline_from_instances(self) -> None:
|
|
162
|
+
result = (A() | B() | C()).run({"text": " hello "})
|
|
163
|
+
|
|
164
|
+
self.assertEqual(result.output, "RESULT=HELLO")
|
|
165
|
+
self.assertEqual(
|
|
166
|
+
result.state["trace"],
|
|
167
|
+
[
|
|
168
|
+
"a.before",
|
|
169
|
+
"a.run",
|
|
170
|
+
"a.after",
|
|
171
|
+
"b.before",
|
|
172
|
+
"b.run",
|
|
173
|
+
"b.after",
|
|
174
|
+
"c.before",
|
|
175
|
+
"c.run",
|
|
176
|
+
"c.after",
|
|
177
|
+
],
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def test_pipe_operator_does_not_mutate_existing_pipeline(self) -> None:
|
|
181
|
+
base = Pipeline("request")
|
|
182
|
+
extended = base | A() | B()
|
|
183
|
+
|
|
184
|
+
self.assertEqual(base.steps, ())
|
|
185
|
+
self.assertEqual(len(extended.steps), 2)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
if __name__ == "__main__":
|
|
189
|
+
unittest.main()
|