simpleflow 0.35.0__tar.gz → 0.37.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. {simpleflow-0.35.0 → simpleflow-0.37.0}/PKG-INFO +6 -7
  2. {simpleflow-0.35.0 → simpleflow-0.37.0}/pyproject.toml +47 -19
  3. simpleflow-0.37.0/simpleflow/__init__.py +16 -0
  4. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/activity.py +2 -1
  5. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/execute.py +4 -3
  6. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/format.py +24 -1
  7. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/history.py +2 -1
  8. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/process/_named_mixin.py +6 -0
  9. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/constants.py +4 -0
  10. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/submittable.py +24 -8
  11. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/utils.py +1 -1
  12. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/workflow.py +65 -4
  13. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/executor.py +5 -2
  14. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/exceptions.py +3 -2
  15. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/base.py +1 -7
  16. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/task.py +1 -8
  17. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/workflow.py +1 -6
  18. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/history/base.py +2 -1
  19. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/stats/pretty.py +2 -1
  20. simpleflow-0.35.0/simpleflow/__init__.py +0 -10
  21. {simpleflow-0.35.0 → simpleflow-0.37.0}/.gitignore +0 -0
  22. {simpleflow-0.35.0 → simpleflow-0.37.0}/LICENSE +0 -0
  23. {simpleflow-0.35.0 → simpleflow-0.37.0}/README.md +0 -0
  24. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/__main__.py +0 -0
  25. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/_decorators.py +0 -0
  26. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/base.py +0 -0
  27. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/boto3_utils.py +0 -0
  28. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/canvas.py +0 -0
  29. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/command.py +0 -0
  30. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/constants.py +0 -0
  31. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/__init__.py +0 -0
  32. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/by_module.py +0 -0
  33. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/dry_run.py +0 -0
  34. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/dynamic_dispatcher.py +0 -0
  35. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/exceptions.py +0 -0
  36. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/dispatch/from_task_registry.py +0 -0
  37. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/download.py +0 -0
  38. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/exceptions.py +0 -0
  39. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/executor.py +0 -0
  40. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/futures.py +0 -0
  41. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/job/__init__.py +0 -0
  42. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/local/__init__.py +0 -0
  43. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/local/executor.py +0 -0
  44. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/log.py +0 -0
  45. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/logging_context.py +0 -0
  46. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/marker.py +0 -0
  47. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/metrology.py +0 -0
  48. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/process/__init__.py +0 -0
  49. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/process/_supervisor.py +0 -0
  50. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/registry.py +0 -0
  51. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/runtime/__init__.py +0 -0
  52. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/settings/__init__.py +0 -0
  53. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/settings/base.py +0 -0
  54. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/settings/default.py +0 -0
  55. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/signal.py +0 -0
  56. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/__init__.py +0 -0
  57. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/step/tasks.py +0 -0
  58. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/storage.py +0 -0
  59. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/__init__.py +0 -0
  60. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/constants.py +0 -0
  61. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/helpers.py +0 -0
  62. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/__init__.py +0 -0
  63. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/actors/__init__.py +0 -0
  64. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/actors/core.py +0 -0
  65. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/actors/decider.py +0 -0
  66. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/actors/worker.py +0 -0
  67. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/constants.py +0 -0
  68. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/core.py +0 -0
  69. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/__init__.py +0 -0
  70. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/activity.py +0 -0
  71. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/base.py +0 -0
  72. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/__init__.py +0 -0
  73. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/base.py +0 -0
  74. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/marker.py +0 -0
  75. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/task.py +0 -0
  76. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/timer.py +0 -0
  77. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/decision/workflow.py +0 -0
  78. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/domain.py +0 -0
  79. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/__init__.py +0 -0
  80. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/compiler.py +0 -0
  81. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/factory.py +0 -0
  82. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/marker.py +0 -0
  83. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/event/timer.py +0 -0
  84. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/history/__init__.py +0 -0
  85. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/history/builder.py +0 -0
  86. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/models/workflow.py +0 -0
  87. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/__init__.py +0 -0
  88. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/activity.py +0 -0
  89. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/base.py +0 -0
  90. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/domain.py +0 -0
  91. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/history.py +0 -0
  92. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/querysets/workflow.py +0 -0
  93. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/responses/__init__.py +0 -0
  94. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/responses/_base.py +0 -0
  95. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/settings.py +0 -0
  96. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/mapper/utils.py +0 -0
  97. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/__init__.py +0 -0
  98. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/decider/__init__.py +0 -0
  99. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/decider/base.py +0 -0
  100. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/decider/command.py +0 -0
  101. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/decider/helpers.py +0 -0
  102. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/poller.py +0 -0
  103. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/worker/__init__.py +0 -0
  104. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/worker/base.py +0 -0
  105. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/process/worker/command.py +0 -0
  106. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/stats/__init__.py +0 -0
  107. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/stats/base.py +0 -0
  108. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/task.py +0 -0
  109. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/swf/utils.py +0 -0
  110. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/task.py +0 -0
  111. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/utils/__init__.py +0 -0
  112. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/utils/_dict.py +0 -0
  113. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/utils/_json.py +0 -0
  114. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/utils/retry.py +0 -0
  115. {simpleflow-0.35.0 → simpleflow-0.37.0}/simpleflow/workflow.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: simpleflow
3
- Version: 0.35.0
3
+ Version: 0.37.0
4
4
  Summary: Python library for dataflow programming with Amazon SWF
5
5
  Project-URL: documentation, https://botify-labs.github.io/simpleflow
6
6
  Project-URL: repository, https://github.com/botify-labs/simpleflow
@@ -15,19 +15,16 @@ Classifier: Intended Audience :: Developers
15
15
  Classifier: License :: OSI Approved :: MIT License
16
16
  Classifier: Natural Language :: English
17
17
  Classifier: Programming Language :: Python :: 3
18
- Classifier: Programming Language :: Python :: 3.7
19
- Classifier: Programming Language :: Python :: 3.8
20
- Classifier: Programming Language :: Python :: 3.9
21
18
  Classifier: Programming Language :: Python :: 3.10
22
19
  Classifier: Programming Language :: Python :: 3.11
23
20
  Classifier: Programming Language :: Python :: 3.12
24
21
  Classifier: Programming Language :: Python :: 3.13
25
- Requires-Python: >=3.7
22
+ Classifier: Programming Language :: Python :: 3.14
23
+ Requires-Python: >=3.10
26
24
  Requires-Dist: attrs
27
25
  Requires-Dist: boto3>=1.28.20
28
26
  Requires-Dist: click
29
27
  Requires-Dist: dill>=0.3.6
30
- Requires-Dist: diskcache>=4.1.0
31
28
  Requires-Dist: jinja2
32
29
  Requires-Dist: lazy-object-proxy
33
30
  Requires-Dist: lockfile>=0.9.1
@@ -37,6 +34,8 @@ Requires-Dist: psutil
37
34
  Requires-Dist: pytz
38
35
  Requires-Dist: setproctitle
39
36
  Requires-Dist: tabulate<1.0.0,>=0.8.2
37
+ Provides-Extra: diskcache
38
+ Requires-Dist: diskcache>=4.1.0; extra == 'diskcache'
40
39
  Description-Content-Type: text/markdown
41
40
 
42
41
  Simpleflow
@@ -1,19 +1,21 @@
1
1
  [build-system]
2
- requires = ["hatchling"]
2
+ requires = ["hatchling", "uv-dynamic-versioning"]
3
3
  build-backend = "hatchling.build"
4
4
 
5
+ [tool.hatch.version]
6
+ source = "uv-dynamic-versioning"
7
+
8
+ [tool.uv-dynamic-versioning]
9
+ pattern = "default-unprefixed"
10
+
5
11
  [project]
6
12
  name = "simpleflow"
7
13
  description = "Python library for dataflow programming with Amazon SWF"
8
14
  readme = "README.md"
9
- requires-python = ">=3.7"
15
+ requires-python = ">=3.10"
10
16
  license = "MIT"
11
- authors = [
12
- { name = "Greg Leclercq" },
13
- ]
14
- maintainers = [
15
- { "email" = "tech@botify.com" },
16
- ]
17
+ authors = [{ name = "Greg Leclercq" }]
18
+ maintainers = [{ "email" = "tech@botify.com" }]
17
19
  keywords = ["simpleflow", "amazon swf", "simple workflow"]
18
20
  classifiers = [
19
21
  "Development Status :: 4 - Beta",
@@ -21,13 +23,11 @@ classifiers = [
21
23
  "License :: OSI Approved :: MIT License",
22
24
  "Natural Language :: English",
23
25
  "Programming Language :: Python :: 3",
24
- "Programming Language :: Python :: 3.7",
25
- "Programming Language :: Python :: 3.8",
26
- "Programming Language :: Python :: 3.9",
27
26
  "Programming Language :: Python :: 3.10",
28
27
  "Programming Language :: Python :: 3.11",
29
28
  "Programming Language :: Python :: 3.12",
30
29
  "Programming Language :: Python :: 3.13",
30
+ "Programming Language :: Python :: 3.14",
31
31
  ]
32
32
  dynamic = ["version"]
33
33
 
@@ -35,7 +35,6 @@ dependencies = [
35
35
  "attrs",
36
36
  "boto3>=1.28.20",
37
37
  "dill>=0.3.6",
38
- "diskcache>=4.1.0",
39
38
  "Jinja2",
40
39
  "markupsafe",
41
40
  "multiprocess>=0.70.14",
@@ -48,6 +47,11 @@ dependencies = [
48
47
  "pytz",
49
48
  ]
50
49
 
50
+ [project.optional-dependencies]
51
+ # Jumbo fields disk cache, only used when SIMPLEFLOW_ENABLE_DISK_CACHE is set (deprecated).
52
+ # Optional because diskcache unpickles cache entries: https://osv.dev/vulnerability/CVE-2025-69872
53
+ diskcache = ["diskcache>=4.1.0"]
54
+
51
55
  [project.urls]
52
56
  documentation = "https://botify-labs.github.io/simpleflow"
53
57
  repository = "https://github.com/botify-labs/simpleflow"
@@ -58,6 +62,7 @@ simpleflow = "simpleflow.command:cli"
58
62
 
59
63
  [tool.ruff]
60
64
  line-length = 120
65
+ unsafe-fixes = true
61
66
 
62
67
  [tool.ruff.lint]
63
68
  extend-select = [
@@ -68,8 +73,7 @@ extend-select = [
68
73
  "RUF", # ruff
69
74
  ]
70
75
  allowed-confusables = ["‘", "’"]
71
- extend-ignore = [
72
- ]
76
+ extend-ignore = []
73
77
  ignore = ["E501"]
74
78
 
75
79
  [tool.ruff.lint.isort]
@@ -80,10 +84,35 @@ exclude_dirs = ["/tests", "/examples", "/venv", "/build"]
80
84
  skips = ["B404"]
81
85
 
82
86
  [tool.pytest.ini_options]
83
- addopts = "--doctest-modules --ignore=setup.py --ignore=tasks.py --ignore=docs/ --ignore=build/ --ignore=examples/"
87
+ addopts = "-vv -rfE --no-success-flaky-report --doctest-modules --ignore=setup.py --ignore=tasks.py --ignore=docs/ --ignore=build/ --ignore=examples/"
88
+ env = [
89
+ "PYTHONDONTWRITEBYTECODE=1",
90
+ # The AWS_DEFAULT_REGION parameter determines the region used for SWF.
91
+ # Leaving it to a value different than "us-east-1" would break moto,
92
+ # because moto.swf only mocks calls to us-east-1 region for now.
93
+ "AWS_DEFAULT_REGION=us-east-1",
94
+ # Disable AWS credentials.
95
+ # NB: we set them to wrong values so boto believes they're here, so it
96
+ # issues integration requests later caught by VCR. For more information
97
+ # see tests/integration/README.md
98
+ "AWS_ACCESS_KEY_ID=1234",
99
+ "AWS_SECRET_ACCESS_KEY=1234",
100
+ # Enable various test optimizations
101
+ "SIMPLEFLOW_ENV=test",
102
+ "SIMPLEFLOW_CLEANUP_PROCESSES=yes",
103
+ # Simple variable to control the number of connection retries
104
+ "SWF_CONNECTION_RETRIES=1",
105
+ # Don't run real SWF requests if a cassette doesn't match
106
+ # See tests/integration/README.md for instructions about cassettes updating
107
+ "SIMPLEFLOW_VCR_RECORD_MODE=none",
108
+ # Disable jumbo fields
109
+ "SIMPLEFLOW_JUMBO_FIELDS_BUCKET=",
110
+ # setproctitle segfaults in forked children on macOS (py-setproctitle#165)
111
+ "SIMPLEFLOW_DISABLE_SETPROCTITLE=1",
112
+ # Prevent CI from overriding boto configuration
113
+ "BOTO_CONFIG=/dev/null",
114
+ ]
84
115
 
85
- [tool.hatch.version]
86
- path = "simpleflow/__init__.py"
87
116
 
88
117
  [tool.hatch.build]
89
118
  include = ["/simpleflow"]
@@ -102,13 +131,12 @@ dev = [
102
131
  "pytest",
103
132
  "pytest-xdist>=3.2.0",
104
133
  "ruff",
105
- "setuptools<72", # See https://github.com/pypa/setuptools/issues/4519
106
134
  "sphinx",
107
135
  "sphinx_rtd_theme",
108
- "sure",
109
136
  "typing-extensions",
110
137
  "vcrpy",
111
138
  "twine",
139
+ "pytest-env>=1.0.1",
112
140
  ]
113
141
 
114
142
  doc = [
@@ -0,0 +1,16 @@
1
+ from __future__ import annotations
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ from .activity import Activity as Activity
6
+ from .runtime import logger as logger
7
+ from .signal import WaitForSignal as WaitForSignal
8
+ from .workflow import Workflow as Workflow
9
+
10
+ try:
11
+ __version__ = version("simpleflow")
12
+ except PackageNotFoundError:
13
+ # Running from a source checkout without the distribution installed.
14
+ __version__ = "unknown"
15
+ __author__ = "Greg Leclercq, Yves Bastide"
16
+ __license__ = "MIT"
@@ -5,7 +5,8 @@ from typing import TYPE_CHECKING
5
5
  from . import registry, settings
6
6
 
7
7
  if TYPE_CHECKING:
8
- from typing import Any, Callable
8
+ from collections.abc import Callable
9
+ from typing import Any
9
10
 
10
11
 
11
12
  __all__ = [
@@ -155,9 +155,10 @@ def python(
155
155
  tmp_dir = env.get(envname)
156
156
  if tmp_dir:
157
157
  break
158
- with tempfile.TemporaryFile(dir=tmp_dir) as result_fd, tempfile.TemporaryFile(
159
- dir=tmp_dir, buffering=0
160
- ) as error_fd: # TODO when Python 3.7 is dropped: encoding="utf-8", errors="replace"
158
+ with (
159
+ tempfile.TemporaryFile(dir=tmp_dir) as result_fd,
160
+ tempfile.TemporaryFile(dir=tmp_dir, buffering=0) as error_fd,
161
+ ):
161
162
  dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC
162
163
  dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC
163
164
  arguments_json = format_arguments_json(*args, **kwargs)
@@ -1,12 +1,12 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import os
4
+ import warnings
4
5
  from sqlite3 import OperationalError
5
6
  from typing import Any, cast
6
7
  from uuid import uuid4
7
8
 
8
9
  import lazy_object_proxy
9
- from diskcache import Cache
10
10
 
11
11
  from simpleflow import constants, logger, storage
12
12
  from simpleflow.settings import SIMPLEFLOW_ENABLE_DISK_CACHE
@@ -78,6 +78,17 @@ def encode(message: str | None, max_length: int, allow_jumbo_fields: bool = True
78
78
  return message
79
79
 
80
80
 
81
+ DISK_CACHE_DEPRECATION = (
82
+ "SIMPLEFLOW_ENABLE_DISK_CACHE is deprecated and will be removed: its only backend, diskcache, "
83
+ "unpickles cache entries (https://osv.dev/vulnerability/CVE-2025-69872), so anyone who can write to the cache "
84
+ "directory can run code in the process that reads it."
85
+ )
86
+
87
+
88
+ def _warn_disk_cache_deprecated() -> None:
89
+ warnings.warn(DISK_CACHE_DEPRECATION, DeprecationWarning, stacklevel=3)
90
+
91
+
81
92
  def _get_cached(path: str) -> str | None:
82
93
  # 1/ memory cache
83
94
  if path in JUMBO_FIELDS_MEMORY_CACHE:
@@ -85,11 +96,15 @@ def _get_cached(path: str) -> str | None:
85
96
 
86
97
  # 2/ disk cache
87
98
  if SIMPLEFLOW_ENABLE_DISK_CACHE:
99
+ _warn_disk_cache_deprecated()
88
100
  try:
89
101
  # NB: this cache may also be triggered on activity workers, where it's not that
90
102
  # useful. The performance hit should be minimal. To be improved later.
91
103
  # NB2: cache has to be lazily instantiated here, cache objects do not survive forks,
92
104
  # see DiskCache docs.
105
+ # Optional extra: simpleflow[diskcache]. Deprecated, see _warn_disk_cache_deprecated.
106
+ from diskcache import Cache
107
+
93
108
  cache = Cache(constants.CACHE_DIR)
94
109
  # generate a dedicated cache key because this cache may be shared with other
95
110
  # features of simpleflow at some point
@@ -97,6 +112,8 @@ def _get_cached(path: str) -> str | None:
97
112
  if cache_key in cache:
98
113
  logger.debug(f"diskcache: getting key={cache_key} from cache_dir={constants.CACHE_DIR}")
99
114
  return cache[cache_key]
115
+ except ImportError:
116
+ logger.warning("diskcache: not installed (pip install simpleflow[diskcache]), skipping cache usage")
100
117
  except OperationalError:
101
118
  logger.warning("diskcache: got an OperationalError, skipping cache usage")
102
119
 
@@ -110,11 +127,17 @@ def _set_cached(path: str, content: str) -> None:
110
127
 
111
128
  # 2/ disk cache
112
129
  if SIMPLEFLOW_ENABLE_DISK_CACHE:
130
+ _warn_disk_cache_deprecated()
113
131
  try:
132
+ # Optional extra: simpleflow[diskcache]. Deprecated, see _warn_disk_cache_deprecated.
133
+ from diskcache import Cache
134
+
114
135
  cache = Cache(constants.CACHE_DIR)
115
136
  cache_key = "jumbo_fields/" + path.split("/")[-1]
116
137
  logger.debug(f"diskcache: setting key={cache_key} on cache_dir={constants.CACHE_DIR}")
117
138
  cache.set(cache_key, content, expire=3 * constants.HOUR)
139
+ except ImportError:
140
+ logger.warning("diskcache: not installed (pip install simpleflow[diskcache]), skipping cache write")
118
141
  except OperationalError:
119
142
  logger.warning("diskcache: got an OperationalError on write, skipping cache write")
120
143
 
@@ -1,7 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import collections
4
- from typing import TYPE_CHECKING, Callable, ClassVar
4
+ from collections.abc import Callable
5
+ from typing import TYPE_CHECKING, ClassVar
5
6
 
6
7
  import simpleflow.swf.mapper.models.history
7
8
  from simpleflow import logger
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import functools
4
+ import os
4
5
 
5
6
  from setproctitle import setproctitle
6
7
 
@@ -61,6 +62,11 @@ class NamedMixin:
61
62
  self.set_process_name()
62
63
 
63
64
  def set_process_name(self):
65
+ # setproctitle segfaults when called in a forked child process on macOS
66
+ # (see https://github.com/dvarrazzo/py-setproctitle/issues/165). Allow
67
+ # disabling it so the test suite doesn't crash spawned workers/deciders.
68
+ if os.environ.get("SIMPLEFLOW_DISABLE_SETPROCTITLE"):
69
+ return
64
70
  klass = self.__class__.__name__
65
71
  properties = []
66
72
  for prop in getattr(self, "_named_mixin_properties", []):
@@ -25,3 +25,7 @@ STEP_ACTIVITY_PARAMS_DEFAULT: dict[str, Any] = {
25
25
  }
26
26
 
27
27
  UNKNOWN_CONTEXT = {"run_id": "unknown", "workflow_id": "unknown", "version": "unknown"}
28
+
29
+ # Prefix of the marker memoizing, in the workflow history, the steps done at the
30
+ # beginning of an execution (only used with ``WorkflowStepMixin.steps_in_decider``).
31
+ STEPS_DONE_MARKER_PREFIX = "step.steps_done"
@@ -17,7 +17,7 @@ from .utils import (
17
17
  )
18
18
 
19
19
  if TYPE_CHECKING:
20
- from typing import Sequence
20
+ from collections.abc import Sequence
21
21
 
22
22
  from simpleflow.base import Submittable
23
23
  from simpleflow.executor import Executor
@@ -57,6 +57,15 @@ class Step(SubmittableContainer):
57
57
  if workflow is None:
58
58
  raise ValueError("Executor has no associated workflow")
59
59
 
60
+ steps_in_decider = getattr(workflow, "steps_in_decider", False)
61
+
62
+ def mark_step_done_in_decider(marker_done):
63
+ # The "completed" marker is recorded in the same decision as the S3 push:
64
+ # once it is in the history, the step is not pushed again on replays.
65
+ if not any(m.name == "log.step" and m.details == marker_done for m in workflow.list_markers(all=True)):
66
+ workflow._mark_step_done_in_decider(self.step_name)
67
+ return workflow.record_marker("log.step", marker_done)
68
+
60
69
  def fn_steps_done(steps_done):
61
70
  marker = {
62
71
  "step": self.step_name,
@@ -81,14 +90,19 @@ class Step(SubmittableContainer):
81
90
  chain += (
82
91
  workflow.record_marker("log.step", marker),
83
92
  self.activities,
84
- (
85
- activity.Activity(MarkStepDoneTask, **workflow._get_step_activity_params()),
86
- workflow.get_step_bucket(),
87
- workflow.get_step_path_prefix(),
88
- self.step_name,
89
- ),
90
- workflow.record_marker("log.step", marker_done),
91
93
  )
94
+ if steps_in_decider:
95
+ chain.append(FuncGroup(mark_step_done_in_decider, marker_done))
96
+ else:
97
+ chain += (
98
+ (
99
+ activity.Activity(MarkStepDoneTask, **workflow._get_step_activity_params()),
100
+ workflow.get_step_bucket(),
101
+ workflow.get_step_path_prefix(),
102
+ self.step_name,
103
+ ),
104
+ workflow.record_marker("log.step", marker_done),
105
+ )
92
106
  else:
93
107
  marker["status"] = "skipped"
94
108
  if step_is_skipped_by_force(self.step_name, skipped_steps):
@@ -108,6 +122,8 @@ class Step(SubmittableContainer):
108
122
  chain.bubbles_exception_on_failure = self.bubbles_exception_on_failure
109
123
  return chain
110
124
 
125
+ if steps_in_decider:
126
+ return workflow.submit(fn_steps_done(workflow._get_steps_done_in_decider()))
111
127
  return workflow.submit(
112
128
  Chain(
113
129
  workflow.get_steps_done_activity(),
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  from typing import TYPE_CHECKING
4
4
 
5
5
  if TYPE_CHECKING:
6
- from typing import Sequence
6
+ from collections.abc import Sequence
7
7
 
8
8
 
9
9
  def should_force_step(step_name: str, force_steps: Sequence[str]) -> bool:
@@ -2,21 +2,29 @@ from __future__ import annotations
2
2
 
3
3
  import abc
4
4
  import copy
5
+ import hashlib
5
6
  import os
6
7
  from collections import defaultdict
7
8
  from typing import TYPE_CHECKING
8
9
 
9
- from simpleflow import activity, settings, task
10
+ from simpleflow import activity, constants, settings, task
11
+ from simpleflow.utils import json_dumps, retry
10
12
 
11
- from .constants import STEP_ACTIVITY_PARAMS_DEFAULT
13
+ from .constants import STEP_ACTIVITY_PARAMS_DEFAULT, STEPS_DONE_MARKER_PREFIX
12
14
  from .submittable import Step
13
- from .tasks import GetStepsDoneTask
15
+ from .tasks import GetStepsDoneTask, MarkStepDoneTask
14
16
 
15
17
  if TYPE_CHECKING:
16
- from typing import Any, Sequence
18
+ from collections.abc import Sequence
19
+ from typing import Any
17
20
 
18
21
 
19
22
  class WorkflowStepMixin(metaclass=abc.ABCMeta):
23
+ # Opt-in: list and mark the steps done from the decider instead of
24
+ # GetStepsDoneTask/MarkStepDoneTask activities. The S3 listing is done once
25
+ # per execution and memoized in a marker so that replays stay deterministic.
26
+ steps_in_decider: bool = False
27
+
20
28
  def get_step_bucket(self) -> str:
21
29
  """
22
30
  Return the S3 bucket where to store the steps files.
@@ -109,3 +117,56 @@ class WorkflowStepMixin(metaclass=abc.ABCMeta):
109
117
  Return the list of steps done.
110
118
  """
111
119
  return self.submit(self.get_steps_done_activity()).result
120
+
121
+ def _get_steps_done_in_decider(self) -> list[str]:
122
+ """
123
+ Return the list of steps done, as listed on S3 the first time it is
124
+ needed in this execution.
125
+
126
+ The listing is recorded in a marker, then read back from the history on
127
+ the next replays: the steps marked done later in this execution don't
128
+ change the decisions already taken.
129
+ """
130
+ bucket = self.get_step_bucket()
131
+ path = self.get_step_path_prefix()
132
+ location_hash = hashlib.md5(f"{bucket}/{path}".encode()).hexdigest() # nosec
133
+ marker_name = f"{STEPS_DONE_MARKER_PREFIX}.{location_hash}"
134
+
135
+ # Cache for the current replay only: the workflow instance may be reused
136
+ # across replays, whereas the executor builds a new history for each one.
137
+ history = self.executor._history
138
+ if getattr(self, "_steps_done_snapshots", (None,))[0] is not history:
139
+ self._steps_done_snapshots: tuple[Any, dict[str, list[str]]] = (history, {})
140
+ snapshots = self._steps_done_snapshots[1]
141
+ if marker_name not in snapshots:
142
+ event = self.get_event_details("marker", marker_name)
143
+ if event is not None:
144
+ steps_done = event["details"]["steps_done"]
145
+ else:
146
+ steps_done = _execute_with_retry(GetStepsDoneTask(bucket, path))
147
+ details = {"bucket": bucket, "path": path, "steps_done": steps_done}
148
+ if len(json_dumps(details)) > constants.MAX_DETAILS_LENGTH:
149
+ raise ValueError(
150
+ f"Too many steps done in {bucket}/{path} to record them in a marker;"
151
+ " disable steps_in_decider for this workflow"
152
+ )
153
+ # Not waited for: the marker is recorded along with the other decisions.
154
+ self.submit(self.record_marker(marker_name, details))
155
+ snapshots[marker_name] = steps_done
156
+ return snapshots[marker_name]
157
+
158
+ def _mark_step_done_in_decider(self, step_name: str) -> None:
159
+ """
160
+ Push the step file on S3 from the decider.
161
+ """
162
+ mark_step_done = MarkStepDoneTask(self.get_step_bucket(), self.get_step_path_prefix(), step_name)
163
+ run_context = self.get_run_context()
164
+ if run_context.get("run_id"):
165
+ mark_step_done.context = run_context
166
+ _execute_with_retry(mark_step_done)
167
+
168
+
169
+ @retry.with_delay(nb_times=3, delay=retry.exponential)
170
+ def _execute_with_retry(a_task: GetStepsDoneTask | MarkStepDoneTask) -> Any:
171
+ # An exception in the decider fails the workflow: retry transient S3 errors.
172
+ return a_task.execute()
@@ -6,7 +6,8 @@ import inspect
6
6
  import json
7
7
  import re
8
8
  import traceback
9
- from typing import TYPE_CHECKING, Any, Callable, ClassVar
9
+ from collections.abc import Callable
10
+ from typing import TYPE_CHECKING, Any, ClassVar
10
11
 
11
12
  import multiprocess
12
13
 
@@ -1259,7 +1260,9 @@ class Executor(executor.Executor):
1259
1260
 
1260
1261
  def list_markers(self, all=False):
1261
1262
  if all:
1262
- return [Marker(m["name"], format.decode(m["details"])) for ml in self._history.markers.values() for m in ml]
1263
+ return [
1264
+ Marker(m["name"], format.decode(m.get("details"))) for ml in self._history.markers.values() for m in ml
1265
+ ]
1263
1266
  rc = []
1264
1267
  for ml in self._history.markers.values():
1265
1268
  m = ml[-1]
@@ -1,9 +1,10 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import re
4
- from collections.abc import Sequence
4
+ from collections.abc import Callable, Sequence
5
5
  from functools import partial, wraps
6
- from typing import Any, Callable, Pattern
6
+ from re import Pattern
7
+ from typing import Any
7
8
 
8
9
  from botocore.exceptions import ClientError
9
10
 
@@ -1,13 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
- import sys
4
3
  from datetime import datetime
5
- from typing import Any
6
-
7
- if sys.version_info < (3, 8):
8
- from typing_extensions import TypedDict
9
- else:
10
- from typing import TypedDict
4
+ from typing import Any, TypedDict
11
5
 
12
6
  import pytz
13
7
 
@@ -1,13 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- import sys
4
-
5
- if sys.version_info < (3, 8):
6
- from typing_extensions import TypedDict
7
- else:
8
- from typing import ClassVar, TypedDict
9
-
10
- from typing import TYPE_CHECKING, Any
3
+ from typing import TYPE_CHECKING, Any, ClassVar, TypedDict
11
4
 
12
5
  from simpleflow.swf.mapper.models.event.base import Event, TaskList
13
6
  from simpleflow.swf.mapper.models.event.compiler import CompiledEvent
@@ -1,11 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- import sys
4
-
5
- if sys.version_info < (3, 8):
6
- from typing_extensions import TypedDict
7
- else:
8
- from typing import ClassVar, TypedDict
3
+ from typing import ClassVar, TypedDict
9
4
 
10
5
  from simpleflow.swf.mapper.models.event.base import Event, TaskList
11
6
  from simpleflow.swf.mapper.models.event.compiler import CompiledEvent
@@ -1,7 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
+ from collections.abc import Iterator
3
4
  from itertools import groupby
4
- from typing import Any, Iterator
5
+ from typing import Any
5
6
 
6
7
  from simpleflow.swf.mapper.models.event.base import Event
7
8
  from simpleflow.swf.mapper.models.event.compiler import CompiledEvent
@@ -1,10 +1,11 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import operator
4
+ from collections.abc import Sequence
4
5
  from datetime import datetime
5
6
  from functools import partial, wraps
6
7
  from itertools import chain
7
- from typing import TYPE_CHECKING, Any, Sequence
8
+ from typing import TYPE_CHECKING, Any
8
9
 
9
10
  from tabulate import tabulate
10
11
 
@@ -1,10 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from .activity import Activity # NOQA
4
- from .runtime import logger # NOQA
5
- from .signal import WaitForSignal # NOQA
6
- from .workflow import Workflow # NOQA
7
-
8
- __version__ = "0.35.0"
9
- __author__ = "Greg Leclercq"
10
- __license__ = "MIT"
File without changes
File without changes
File without changes