aboutcode.pipeline 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,20 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: aboutcode.pipeline
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: AboutCode Pipeline library. Execute code in steps.
5
5
  Keywords: open source,pipeline,scancode
6
6
  Author-email: "nexB. Inc. and others" <info@aboutcode.org>
7
- Requires-Python: >=3.9
7
+ Requires-Python: >=3.11
8
8
  Description-Content-Type: text/markdown
9
9
  Classifier: Development Status :: 4 - Beta
10
10
  Classifier: Intended Audience :: Developers
11
11
  Classifier: Programming Language :: Python :: 3
12
12
  Classifier: Programming Language :: Python :: 3 :: Only
13
- Classifier: Programming Language :: Python :: 3.9
14
- Classifier: Programming Language :: Python :: 3.10
15
13
  Classifier: Programming Language :: Python :: 3.11
16
14
  Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
17
+ Classifier: Programming Language :: Python :: 3.15
17
18
  Classifier: Topic :: Software Development
18
19
  Classifier: Topic :: Utilities
19
20
  Project-URL: Changelog, https://github.com/aboutcode-org/scancode.io/tree/main/aboutcode/pipeline/CHANGELOG.md
@@ -26,7 +27,7 @@ Project-URL: Repository, https://github.com/aboutcode-org/scancode.io/tree/main/
26
27
 
27
28
  Define and run pipelines.
28
29
 
29
- ### Install
30
+ ## Install
30
31
 
31
32
  ```bash
32
33
  pip install aboutcode.pipeline
@@ -37,6 +38,7 @@ pip install aboutcode.pipeline
37
38
  ```python
38
39
  from aboutcode.pipeline import BasePipeline
39
40
 
41
+
40
42
  class PrintMessages(BasePipeline):
41
43
  @classmethod
42
44
  def steps(cls):
@@ -45,6 +47,7 @@ class PrintMessages(BasePipeline):
45
47
  def step1(self):
46
48
  print("Message from step1")
47
49
 
50
+
48
51
  PrintMessages().execute()
49
52
  ```
50
53
 
@@ -77,3 +80,29 @@ run = PrintMessages(selected_steps=["step1"])
77
80
  exitcode, error = run.execute()
78
81
  ```
79
82
 
83
+ ### Post steps
84
+
85
+ Post steps run at the end of the pipeline, even when a previous step failed.
86
+ Use them for cleanup. A post step must not rely on the state set by the other steps,
87
+ as they may have failed before setting it.
88
+
89
+ ```python
90
+ from aboutcode.pipeline import BasePipeline
91
+
92
+
93
+ class PrintMessages(BasePipeline):
94
+ @classmethod
95
+ def steps(cls):
96
+ return (cls.step1,)
97
+
98
+ @classmethod
99
+ def get_post_steps(cls):
100
+ return (cls.cleanup,)
101
+
102
+ def step1(self):
103
+ raise ValueError("Failure in step1")
104
+
105
+ def cleanup(self):
106
+ print("Cleanup runs after the step1 failure")
107
+ ```
108
+
@@ -0,0 +1,32 @@
1
+ # Changelog
2
+
3
+ ## Release 0.3.0 (November 16, 2026)
4
+
5
+ * Add the ``get_post_steps()`` class method to declare steps that always run at the
6
+ end of a pipeline, even when a previous step failed.
7
+ [Issue #2310](https://github.com/aboutcode-org/scancode.io/issues/2310)
8
+ * Include the groups of the initial and post steps in ``get_available_groups()``.
9
+ * Fix ``humanize_time()`` returning both days and hours above one day.
10
+ * ``get_initial_steps()`` and ``get_post_steps()`` return an empty tuple by default.
11
+ Overrides returning ``None`` are still supported.
12
+ * Breaking: Remove the deprecated ``group`` decorator.
13
+ Migration: Use the ``optional_step`` decorator.
14
+ * Remove the unused ``LoopProgress.get_eta()`` method.
15
+ * Require Python 3.11 or later, and support Python 3.11 to 3.15.
16
+
17
+ ## Release 0.2.1 (February 24, 2025)
18
+
19
+ * Include the ``optional_step`` steps in the ``get_graph()`` list.
20
+ [Issue #1599](https://github.com/aboutcode-org/scancode.io/issues/1599)
21
+
22
+ ## Release 0.2.0 (November 21, 2024)
23
+
24
+ * Refactor the ``group`` decorator for pipeline optional steps as ``optional_step``.
25
+ The steps decorated as optional are not included by default anymore.
26
+ Migration: Use the ``optional_step`` decorator in place of the deprecated ``group``.
27
+ [Issue #1442](https://github.com/nexB/scancode.io/issues/1442)
28
+
29
+ ## Release 0.1.0 (August 9, 2024)
30
+
31
+ * Initial release of the `aboutcode.pipeline` library.
32
+ [Issue #1351](https://github.com/nexB/scancode.io/issues/1351)
@@ -2,7 +2,7 @@
2
2
 
3
3
  Define and run pipelines.
4
4
 
5
- ### Install
5
+ ## Install
6
6
 
7
7
  ```bash
8
8
  pip install aboutcode.pipeline
@@ -13,6 +13,7 @@ pip install aboutcode.pipeline
13
13
  ```python
14
14
  from aboutcode.pipeline import BasePipeline
15
15
 
16
+
16
17
  class PrintMessages(BasePipeline):
17
18
  @classmethod
18
19
  def steps(cls):
@@ -21,6 +22,7 @@ class PrintMessages(BasePipeline):
21
22
  def step1(self):
22
23
  print("Message from step1")
23
24
 
25
+
24
26
  PrintMessages().execute()
25
27
  ```
26
28
 
@@ -52,3 +54,29 @@ exitcode, error = run.execute()
52
54
  run = PrintMessages(selected_steps=["step1"])
53
55
  exitcode, error = run.execute()
54
56
  ```
57
+
58
+ ### Post steps
59
+
60
+ Post steps run at the end of the pipeline, even when a previous step failed.
61
+ Use them for cleanup. A post step must not rely on the state set by the other steps,
62
+ as they may have failed before setting it.
63
+
64
+ ```python
65
+ from aboutcode.pipeline import BasePipeline
66
+
67
+
68
+ class PrintMessages(BasePipeline):
69
+ @classmethod
70
+ def steps(cls):
71
+ return (cls.step1,)
72
+
73
+ @classmethod
74
+ def get_post_steps(cls):
75
+ return (cls.cleanup,)
76
+
77
+ def step1(self):
78
+ raise ValueError("Failure in step1")
79
+
80
+ def cleanup(self):
81
+ print("Cleanup runs after the step1 failure")
82
+ ```
@@ -22,7 +22,6 @@
22
22
 
23
23
  import logging
24
24
  import traceback
25
- import warnings
26
25
  from datetime import datetime
27
26
  from datetime import timezone
28
27
  from pydoc import getdoc
@@ -31,7 +30,7 @@ from timeit import default_timer as timer
31
30
 
32
31
  module_logger = logging.getLogger(__name__)
33
32
 
34
- __version__ = "0.2.0"
33
+ __version__ = "0.3.0"
35
34
 
36
35
 
37
36
  class PipelineDefinition:
@@ -49,26 +48,36 @@ class PipelineDefinition:
49
48
  def steps(cls):
50
49
  raise NotImplementedError
51
50
 
51
+ @classmethod
52
+ def get_all_steps(cls):
53
+ """
54
+ Return the initial steps, the steps defined in the ``steps`` class method,
55
+ and the post steps, including the steps decorated with ``optional_step``.
56
+ """
57
+ if not callable(cls.steps):
58
+ raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
59
+
60
+ # Overrides may return None when there is no step to add, as the default did
61
+ # before 0.3.0. Keep supporting it: a TypeError here would break the listing
62
+ # of every pipeline in ScanCode.io, not only the one returning None.
63
+ initial_steps = cls.get_initial_steps() or ()
64
+ post_steps = cls.get_post_steps() or ()
65
+ return (*initial_steps, *cls.steps(), *post_steps)
66
+
52
67
  @classmethod
53
68
  def get_steps(cls, groups=None):
54
69
  """
55
- Return the list of steps defined in the ``steps`` class method.
70
+ Return the tuple of the initial steps, the steps defined in the ``steps``
71
+ class method, and the post steps.
56
72
 
57
73
  By default, all steps decorated with ``optional_step`` are not included.
58
74
  A list of optional steps can be included using the ``groups`` parameter.
59
75
  """
60
- if not callable(cls.steps):
61
- raise TypeError("Use a ``steps(cls)`` classmethod to declare the steps.")
62
-
63
- steps = cls.steps()
64
76
  groups = groups or []
65
77
 
66
- if initial_steps := cls.get_initial_steps():
67
- steps = (*initial_steps, *steps)
68
-
69
78
  steps = tuple(
70
79
  step
71
- for step in steps
80
+ for step in cls.get_all_steps()
72
81
  if not getattr(step, "groups", [])
73
82
  or set(getattr(step, "groups")).intersection(groups)
74
83
  )
@@ -81,7 +90,15 @@ class PipelineDefinition:
81
90
  Return a tuple of extra initial steps to be run at the start of the pipeline
82
91
  execution.
83
92
  """
84
- return
93
+ return ()
94
+
95
+ @classmethod
96
+ def get_post_steps(cls):
97
+ """
98
+ Return a tuple of extra steps to be run at the end of the pipeline
99
+ execution, even when a previous step failed.
100
+ """
101
+ return ()
85
102
 
86
103
  @classmethod
87
104
  def get_doc(cls):
@@ -97,7 +114,7 @@ class PipelineDefinition:
97
114
  "doc": getdoc(step),
98
115
  "groups": getattr(step, "groups", []),
99
116
  }
100
- for step in cls.get_steps()
117
+ for step in cls.get_steps(groups=cls.get_available_groups())
101
118
  ]
102
119
 
103
120
  @classmethod
@@ -123,7 +140,7 @@ class PipelineDefinition:
123
140
  return sorted(
124
141
  set(
125
142
  group_name
126
- for step in cls.steps()
143
+ for step in cls.get_all_steps()
127
144
  for group_name in getattr(step, "groups", [])
128
145
  )
129
146
  )
@@ -177,16 +194,24 @@ class PipelineRun:
177
194
  return output
178
195
 
179
196
  def execute(self):
180
- """Execute each steps in the order defined on this pipeline class."""
197
+ """
198
+ Execute each steps in the order defined on this pipeline class.
199
+ After a failure, only the post steps are executed.
200
+ """
181
201
  self.log(f"Pipeline [{self.pipeline_name}] starting")
182
202
 
183
203
  steps = self.pipeline_class.get_steps(groups=self.selected_groups)
204
+ post_steps = self.pipeline_class.get_post_steps() or ()
184
205
  steps_count = len(steps)
185
206
  pipeline_start_time = timer()
207
+ failure_output = ""
186
208
 
187
209
  for current_index, step in enumerate(steps, start=1):
188
210
  step_name = step.__name__
189
211
 
212
+ if failure_output and step not in post_steps:
213
+ continue
214
+
190
215
  if self.selected_steps and step_name not in self.selected_steps:
191
216
  self.log(f"Step [{step_name}] skipped")
192
217
  continue
@@ -198,12 +223,19 @@ class PipelineRun:
198
223
  try:
199
224
  step(self)
200
225
  except Exception as exception:
201
- self.log("Pipeline failed")
202
- return 1, self.output_from_exception(exception)
226
+ self.log(f"Step [{step_name}] failed")
227
+ # Keep the first failure, the root cause, as the pipeline output.
228
+ if not failure_output:
229
+ failure_output = self.output_from_exception(exception)
230
+ continue
203
231
 
204
232
  step_run_time = timer() - step_start_time
205
233
  self.log(f"Step [{step_name}] completed in {humanize_time(step_run_time)}")
206
234
 
235
+ if failure_output:
236
+ self.log("Pipeline failed")
237
+ return 1, failure_output
238
+
207
239
  # Reset the `current_step` field on completion
208
240
  self.set_current_step("")
209
241
  pipeline_run_time = timer() - pipeline_start_time
@@ -232,24 +264,13 @@ def optional_step(*groups):
232
264
  return decorator
233
265
 
234
266
 
235
- def group(*groups):
236
- """Backward compatibility."""
237
- warnings.warn(
238
- "The `group` decorator is deprecated and will be "
239
- "removed in a future release. Use `optional_step` instead.",
240
- DeprecationWarning,
241
- stacklevel=2,
242
- )
243
- return optional_step(*groups)
244
-
245
-
246
267
  def humanize_time(seconds):
247
268
  """Convert the provided ``seconds`` number into human-readable time."""
248
269
  message = f"{seconds:.0f} seconds"
249
270
 
250
271
  if seconds > 86400:
251
272
  message += f" ({seconds / 86400:.1f} days)"
252
- if seconds > 3600:
273
+ elif seconds > 3600:
253
274
  message += f" ({seconds / 3600:.1f} hours)"
254
275
  elif seconds > 60:
255
276
  message += f" ({seconds / 60:.1f} minutes)"
@@ -283,10 +304,6 @@ class LoopProgress:
283
304
  self.last_logged_progress = 0
284
305
  self.current_iteration = 0
285
306
 
286
- def get_eta(self, current_progress):
287
- run_time = timer() - self.start_time
288
- return round(run_time / current_progress * (100 - current_progress))
289
-
290
307
  @property
291
308
  def current_progress(self):
292
309
  return int((self.current_iteration / self.total_iterations) * 100)
@@ -4,11 +4,11 @@ build-backend = "flot.buildapi"
4
4
 
5
5
  [project]
6
6
  name = "aboutcode.pipeline"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "AboutCode Pipeline library. Execute code in steps."
9
9
  license = { text = "Apache-2.0" }
10
10
  readme = "aboutcode/pipeline/README.md"
11
- requires-python = ">=3.9"
11
+ requires-python = ">=3.11"
12
12
  authors = [ { name = "nexB. Inc. and others", email = "info@aboutcode.org" } ]
13
13
  keywords = [
14
14
  "open source",
@@ -20,10 +20,11 @@ classifiers = [
20
20
  "Intended Audience :: Developers",
21
21
  "Programming Language :: Python :: 3",
22
22
  "Programming Language :: Python :: 3 :: Only",
23
- "Programming Language :: Python :: 3.9",
24
- "Programming Language :: Python :: 3.10",
25
23
  "Programming Language :: Python :: 3.11",
26
24
  "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Programming Language :: Python :: 3.14",
27
+ "Programming Language :: Python :: 3.15",
27
28
  "Topic :: Software Development",
28
29
  "Topic :: Utilities",
29
30
  ]
@@ -1,13 +0,0 @@
1
- # Changelog
2
-
3
- ## Release 0.2.0 (November 21, 2024)
4
-
5
- * Refactor the ``group`` decorator for pipeline optional steps as ``optional_step``.
6
- The steps decorated as optional are not included by default anymore.
7
- Migration: Use the ``optional_step`` decorator in place of the deprecated ``group``.
8
- [Issue #1442](https://github.com/nexB/scancode.io/issues/1442)
9
-
10
- ## Release 0.1.0 (August 9, 2024)
11
-
12
- * Initial release of the `aboutcode.pipeline` library.
13
- [Issue #1351](https://github.com/nexB/scancode.io/issues/1351)