aboutcode.pipeline 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/PKG-INFO +34 -5
- aboutcode_pipeline-0.3.0/aboutcode/pipeline/CHANGELOG.md +32 -0
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/aboutcode/pipeline/README.md +29 -1
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/aboutcode/pipeline/__init__.py +50 -33
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/pyproject.toml +5 -4
- aboutcode_pipeline-0.2.0/aboutcode/pipeline/CHANGELOG.md +0 -13
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/LICENSE +0 -0
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/NOTICE +0 -0
- {aboutcode_pipeline-0.2.0 → aboutcode_pipeline-0.3.0}/aboutcode/pipeline/RELEASE.md +0 -0
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: aboutcode.pipeline
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: AboutCode Pipeline library. Execute code in steps.
|
|
5
5
|
Keywords: open source,pipeline,scancode
|
|
6
6
|
Author-email: "nexB. Inc. and others" <info@aboutcode.org>
|
|
7
|
-
Requires-Python: >=3.
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
9
|
Classifier: Development Status :: 4 - Beta
|
|
10
10
|
Classifier: Intended Audience :: Developers
|
|
11
11
|
Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
17
18
|
Classifier: Topic :: Software Development
|
|
18
19
|
Classifier: Topic :: Utilities
|
|
19
20
|
Project-URL: Changelog, https://github.com/aboutcode-org/scancode.io/tree/main/aboutcode/pipeline/CHANGELOG.md
|
|
@@ -26,7 +27,7 @@ Project-URL: Repository, https://github.com/aboutcode-org/scancode.io/tree/main/
|
|
|
26
27
|
|
|
27
28
|
Define and run pipelines.
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
## Install
|
|
30
31
|
|
|
31
32
|
```bash
|
|
32
33
|
pip install aboutcode.pipeline
|
|
@@ -37,6 +38,7 @@ pip install aboutcode.pipeline
|
|
|
37
38
|
```python
|
|
38
39
|
from aboutcode.pipeline import BasePipeline
|
|
39
40
|
|
|
41
|
+
|
|
40
42
|
class PrintMessages(BasePipeline):
|
|
41
43
|
@classmethod
|
|
42
44
|
def steps(cls):
|
|
@@ -45,6 +47,7 @@ class PrintMessages(BasePipeline):
|
|
|
45
47
|
def step1(self):
|
|
46
48
|
print("Message from step1")
|
|
47
49
|
|
|
50
|
+
|
|
48
51
|
PrintMessages().execute()
|
|
49
52
|
```
|
|
50
53
|
|
|
@@ -77,3 +80,29 @@ run = PrintMessages(selected_steps=["step1"])
|
|
|
77
80
|
exitcode, error = run.execute()
|
|
78
81
|
```
|
|
79
82
|
|
|
83
|
+
### Post steps
|
|
84
|
+
|
|
85
|
+
Post steps run at the end of the pipeline, even when a previous step failed.
|
|
86
|
+
Use them for cleanup. A post step must not rely on the state set by the other steps,
|
|
87
|
+
as they may have failed before setting it.
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from aboutcode.pipeline import BasePipeline
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class PrintMessages(BasePipeline):
|
|
94
|
+
@classmethod
|
|
95
|
+
def steps(cls):
|
|
96
|
+
return (cls.step1,)
|
|
97
|
+
|
|
98
|
+
@classmethod
|
|
99
|
+
def get_post_steps(cls):
|
|
100
|
+
return (cls.cleanup,)
|
|
101
|
+
|
|
102
|
+
def step1(self):
|
|
103
|
+
raise ValueError("Failure in step1")
|
|
104
|
+
|
|
105
|
+
def cleanup(self):
|
|
106
|
+
print("Cleanup runs after the step1 failure")
|
|
107
|
+
```
|
|
108
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Release 0.3.0 (November 16, 2026)
|
|
4
|
+
|
|
5
|
+
* Add the ``get_post_steps()`` class method to declare steps that always run at the
|
|
6
|
+
end of a pipeline, even when a previous step failed.
|
|
7
|
+
[Issue #2310](https://github.com/aboutcode-org/scancode.io/issues/2310)
|
|
8
|
+
* Include the groups of the initial and post steps in ``get_available_groups()``.
|
|
9
|
+
* Fix ``humanize_time()`` returning both days and hours above one day.
|
|
10
|
+
* ``get_initial_steps()`` and ``get_post_steps()`` return an empty tuple by default.
|
|
11
|
+
Overrides returning ``None`` are still supported.
|
|
12
|
+
* Breaking: Remove the deprecated ``group`` decorator.
|
|
13
|
+
Migration: Use the ``optional_step`` decorator.
|
|
14
|
+
* Remove the unused ``LoopProgress.get_eta()`` method.
|
|
15
|
+
* Require Python 3.11 or later, and support Python 3.11 to 3.15.
|
|
16
|
+
|
|
17
|
+
## Release 0.2.1 (February 24, 2025)
|
|
18
|
+
|
|
19
|
+
* Include the ``optional_step`` steps in the ``get_graph()`` list.
|
|
20
|
+
[Issue #1599](https://github.com/aboutcode-org/scancode.io/issues/1599)
|
|
21
|
+
|
|
22
|
+
## Release 0.2.0 (November 21, 2024)
|
|
23
|
+
|
|
24
|
+
* Refactor the ``group`` decorator for pipeline optional steps as ``optional_step``.
|
|
25
|
+
The steps decorated as optional are not included by default anymore.
|
|
26
|
+
Migration: Use the ``optional_step`` decorator in place of the deprecated ``group``.
|
|
27
|
+
[Issue #1442](https://github.com/nexB/scancode.io/issues/1442)
|
|
28
|
+
|
|
29
|
+
## Release 0.1.0 (August 9, 2024)
|
|
30
|
+
|
|
31
|
+
* Initial release of the `aboutcode.pipeline` library.
|
|
32
|
+
[Issue #1351](https://github.com/nexB/scancode.io/issues/1351)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Define and run pipelines.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Install
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
8
|
pip install aboutcode.pipeline
|
|
@@ -13,6 +13,7 @@ pip install aboutcode.pipeline
|
|
|
13
13
|
```python
|
|
14
14
|
from aboutcode.pipeline import BasePipeline
|
|
15
15
|
|
|
16
|
+
|
|
16
17
|
class PrintMessages(BasePipeline):
|
|
17
18
|
@classmethod
|
|
18
19
|
def steps(cls):
|
|
@@ -21,6 +22,7 @@ class PrintMessages(BasePipeline):
|
|
|
21
22
|
def step1(self):
|
|
22
23
|
print("Message from step1")
|
|
23
24
|
|
|
25
|
+
|
|
24
26
|
PrintMessages().execute()
|
|
25
27
|
```
|
|
26
28
|
|
|
@@ -52,3 +54,29 @@ exitcode, error = run.execute()
|
|
|
52
54
|
run = PrintMessages(selected_steps=["step1"])
|
|
53
55
|
exitcode, error = run.execute()
|
|
54
56
|
```
|
|
57
|
+
|
|
58
|
+
### Post steps
|
|
59
|
+
|
|
60
|
+
Post steps run at the end of the pipeline, even when a previous step failed.
|
|
61
|
+
Use them for cleanup. A post step must not rely on the state set by the other steps,
|
|
62
|
+
as they may have failed before setting it.
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from aboutcode.pipeline import BasePipeline
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class PrintMessages(BasePipeline):
|
|
69
|
+
@classmethod
|
|
70
|
+
def steps(cls):
|
|
71
|
+
return (cls.step1,)
|
|
72
|
+
|
|
73
|
+
@classmethod
|
|
74
|
+
def get_post_steps(cls):
|
|
75
|
+
return (cls.cleanup,)
|
|
76
|
+
|
|
77
|
+
def step1(self):
|
|
78
|
+
raise ValueError("Failure in step1")
|
|
79
|
+
|
|
80
|
+
def cleanup(self):
|
|
81
|
+
print("Cleanup runs after the step1 failure")
|
|
82
|
+
```
|
|
@@ -22,7 +22,6 @@
|
|
|
22
22
|
|
|
23
23
|
import logging
|
|
24
24
|
import traceback
|
|
25
|
-
import warnings
|
|
26
25
|
from datetime import datetime
|
|
27
26
|
from datetime import timezone
|
|
28
27
|
from pydoc import getdoc
|
|
@@ -31,7 +30,7 @@ from timeit import default_timer as timer
|
|
|
31
30
|
|
|
32
31
|
module_logger = logging.getLogger(__name__)
|
|
33
32
|
|
|
34
|
-
__version__ = "0.
|
|
33
|
+
__version__ = "0.3.0"
|
|
35
34
|
|
|
36
35
|
|
|
37
36
|
class PipelineDefinition:
|
|
@@ -49,26 +48,36 @@ class PipelineDefinition:
|
|
|
49
48
|
def steps(cls):
|
|
50
49
|
raise NotImplementedError
|
|
51
50
|
|
|
51
|
+
@classmethod
|
|
52
|
+
def get_all_steps(cls):
|
|
53
|
+
"""
|
|
54
|
+
Return the initial steps, the steps defined in the ``steps`` class method,
|
|
55
|
+
and the post steps, including the steps decorated with ``optional_step``.
|
|
56
|
+
"""
|
|
57
|
+
if not callable(cls.steps):
|
|
58
|
+
raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
|
|
59
|
+
|
|
60
|
+
# Overrides may return None when there is no step to add, as the default did
|
|
61
|
+
# before 0.3.0. Keep supporting it: a TypeError here would break the listing
|
|
62
|
+
# of every pipeline in ScanCode.io, not only the one returning None.
|
|
63
|
+
initial_steps = cls.get_initial_steps() or ()
|
|
64
|
+
post_steps = cls.get_post_steps() or ()
|
|
65
|
+
return (*initial_steps, *cls.steps(), *post_steps)
|
|
66
|
+
|
|
52
67
|
@classmethod
|
|
53
68
|
def get_steps(cls, groups=None):
|
|
54
69
|
"""
|
|
55
|
-
Return the
|
|
70
|
+
Return the tuple of the initial steps, the steps defined in the ``steps``
|
|
71
|
+
class method, and the post steps.
|
|
56
72
|
|
|
57
73
|
By default, all steps decorated with ``optional_step`` are not included.
|
|
58
74
|
A list of optional steps can be included using the ``groups`` parameter.
|
|
59
75
|
"""
|
|
60
|
-
if not callable(cls.steps):
|
|
61
|
-
raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
|
|
62
|
-
|
|
63
|
-
steps = cls.steps()
|
|
64
76
|
groups = groups or []
|
|
65
77
|
|
|
66
|
-
if initial_steps := cls.get_initial_steps():
|
|
67
|
-
steps = (*initial_steps, *steps)
|
|
68
|
-
|
|
69
78
|
steps = tuple(
|
|
70
79
|
step
|
|
71
|
-
for step in
|
|
80
|
+
for step in cls.get_all_steps()
|
|
72
81
|
if not getattr(step, "groups", [])
|
|
73
82
|
or set(getattr(step, "groups")).intersection(groups)
|
|
74
83
|
)
|
|
@@ -81,7 +90,15 @@ class PipelineDefinition:
|
|
|
81
90
|
Return a tuple of extra initial steps to be run at the start of the pipeline
|
|
82
91
|
execution.
|
|
83
92
|
"""
|
|
84
|
-
return
|
|
93
|
+
return ()
|
|
94
|
+
|
|
95
|
+
@classmethod
|
|
96
|
+
def get_post_steps(cls):
|
|
97
|
+
"""
|
|
98
|
+
Return a tuple of extra steps to be run at the end of the pipeline
|
|
99
|
+
execution, even when a previous step failed.
|
|
100
|
+
"""
|
|
101
|
+
return ()
|
|
85
102
|
|
|
86
103
|
@classmethod
|
|
87
104
|
def get_doc(cls):
|
|
@@ -97,7 +114,7 @@ class PipelineDefinition:
|
|
|
97
114
|
"doc": getdoc(step),
|
|
98
115
|
"groups": getattr(step, "groups", []),
|
|
99
116
|
}
|
|
100
|
-
for step in cls.get_steps()
|
|
117
|
+
for step in cls.get_steps(groups=cls.get_available_groups())
|
|
101
118
|
]
|
|
102
119
|
|
|
103
120
|
@classmethod
|
|
@@ -123,7 +140,7 @@ class PipelineDefinition:
|
|
|
123
140
|
return sorted(
|
|
124
141
|
set(
|
|
125
142
|
group_name
|
|
126
|
-
for step in cls.
|
|
143
|
+
for step in cls.get_all_steps()
|
|
127
144
|
for group_name in getattr(step, "groups", [])
|
|
128
145
|
)
|
|
129
146
|
)
|
|
@@ -177,16 +194,24 @@ class PipelineRun:
|
|
|
177
194
|
return output
|
|
178
195
|
|
|
179
196
|
def execute(self):
|
|
180
|
-
"""
|
|
197
|
+
"""
|
|
198
|
+
Execute each steps in the order defined on this pipeline class.
|
|
199
|
+
After a failure, only the post steps are executed.
|
|
200
|
+
"""
|
|
181
201
|
self.log(f"Pipeline [{self.pipeline_name}] starting")
|
|
182
202
|
|
|
183
203
|
steps = self.pipeline_class.get_steps(groups=self.selected_groups)
|
|
204
|
+
post_steps = self.pipeline_class.get_post_steps() or ()
|
|
184
205
|
steps_count = len(steps)
|
|
185
206
|
pipeline_start_time = timer()
|
|
207
|
+
failure_output = ""
|
|
186
208
|
|
|
187
209
|
for current_index, step in enumerate(steps, start=1):
|
|
188
210
|
step_name = step.__name__
|
|
189
211
|
|
|
212
|
+
if failure_output and step not in post_steps:
|
|
213
|
+
continue
|
|
214
|
+
|
|
190
215
|
if self.selected_steps and step_name not in self.selected_steps:
|
|
191
216
|
self.log(f"Step [{step_name}] skipped")
|
|
192
217
|
continue
|
|
@@ -198,12 +223,19 @@ class PipelineRun:
|
|
|
198
223
|
try:
|
|
199
224
|
step(self)
|
|
200
225
|
except Exception as exception:
|
|
201
|
-
self.log("
|
|
202
|
-
|
|
226
|
+
self.log(f"Step [{step_name}] failed")
|
|
227
|
+
# Keep the first failure, the root cause, as the pipeline output.
|
|
228
|
+
if not failure_output:
|
|
229
|
+
failure_output = self.output_from_exception(exception)
|
|
230
|
+
continue
|
|
203
231
|
|
|
204
232
|
step_run_time = timer() - step_start_time
|
|
205
233
|
self.log(f"Step [{step_name}] completed in {humanize_time(step_run_time)}")
|
|
206
234
|
|
|
235
|
+
if failure_output:
|
|
236
|
+
self.log("Pipeline failed")
|
|
237
|
+
return 1, failure_output
|
|
238
|
+
|
|
207
239
|
# Reset the `current_step` field on completion
|
|
208
240
|
self.set_current_step("")
|
|
209
241
|
pipeline_run_time = timer() - pipeline_start_time
|
|
@@ -232,24 +264,13 @@ def optional_step(*groups):
|
|
|
232
264
|
return decorator
|
|
233
265
|
|
|
234
266
|
|
|
235
|
-
def group(*groups):
|
|
236
|
-
"""Backward compatibility."""
|
|
237
|
-
warnings.warn(
|
|
238
|
-
"The `group` decorator is deprecated and will be "
|
|
239
|
-
"removed in a future release. Use `optional_step` instead.",
|
|
240
|
-
DeprecationWarning,
|
|
241
|
-
stacklevel=2,
|
|
242
|
-
)
|
|
243
|
-
return optional_step(*groups)
|
|
244
|
-
|
|
245
|
-
|
|
246
267
|
def humanize_time(seconds):
|
|
247
268
|
"""Convert the provided ``seconds`` number into human-readable time."""
|
|
248
269
|
message = f"{seconds:.0f} seconds"
|
|
249
270
|
|
|
250
271
|
if seconds > 86400:
|
|
251
272
|
message += f" ({seconds / 86400:.1f} days)"
|
|
252
|
-
|
|
273
|
+
elif seconds > 3600:
|
|
253
274
|
message += f" ({seconds / 3600:.1f} hours)"
|
|
254
275
|
elif seconds > 60:
|
|
255
276
|
message += f" ({seconds / 60:.1f} minutes)"
|
|
@@ -283,10 +304,6 @@ class LoopProgress:
|
|
|
283
304
|
self.last_logged_progress = 0
|
|
284
305
|
self.current_iteration = 0
|
|
285
306
|
|
|
286
|
-
def get_eta(self, current_progress):
|
|
287
|
-
run_time = timer() - self.start_time
|
|
288
|
-
return round(run_time / current_progress * (100 - current_progress))
|
|
289
|
-
|
|
290
307
|
@property
|
|
291
308
|
def current_progress(self):
|
|
292
309
|
return int((self.current_iteration / self.total_iterations) * 100)
|
|
@@ -4,11 +4,11 @@ build-backend = "flot.buildapi"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "aboutcode.pipeline"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "AboutCode Pipeline library. Execute code in steps."
|
|
9
9
|
license = { text = "Apache-2.0" }
|
|
10
10
|
readme = "aboutcode/pipeline/README.md"
|
|
11
|
-
requires-python = ">=3.
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
12
|
authors = [ { name = "nexB. Inc. and others", email = "info@aboutcode.org" } ]
|
|
13
13
|
keywords = [
|
|
14
14
|
"open source",
|
|
@@ -20,10 +20,11 @@ classifiers = [
|
|
|
20
20
|
"Intended Audience :: Developers",
|
|
21
21
|
"Programming Language :: Python :: 3",
|
|
22
22
|
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
-
"Programming Language :: Python :: 3.9",
|
|
24
|
-
"Programming Language :: Python :: 3.10",
|
|
25
23
|
"Programming Language :: Python :: 3.11",
|
|
26
24
|
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
27
|
+
"Programming Language :: Python :: 3.15",
|
|
27
28
|
"Topic :: Software Development",
|
|
28
29
|
"Topic :: Utilities",
|
|
29
30
|
]
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## Release 0.2.0 (November 21, 2024)
|
|
4
|
-
|
|
5
|
-
* Refactor the ``group`` decorator for pipeline optional steps as ``optional_step``.
|
|
6
|
-
The steps decorated as optional are not included by default anymore.
|
|
7
|
-
Migration: Use the ``optional_step`` decorator in place of the deprecated ``group``.
|
|
8
|
-
[Issue #1442](https://github.com/nexB/scancode.io/issues/1442)
|
|
9
|
-
|
|
10
|
-
## Release 0.1.0 (August 9, 2024)
|
|
11
|
-
|
|
12
|
-
* Initial release of the `aboutcode.pipeline` library.
|
|
13
|
-
[Issue #1351](https://github.com/nexB/scancode.io/issues/1351)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|